diff --git a/aicb b/aicb deleted file mode 160000 index 23eec3c48..000000000 --- a/aicb +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 23eec3c48ca2d2d93dd888a4c7b22ab4421e782f diff --git a/aicb/.gitignore b/aicb/.gitignore new file mode 100644 index 000000000..8251627f2 --- /dev/null +++ b/aicb/.gitignore @@ -0,0 +1,4 @@ +*__pycache__* +*local* +.ipynb_checkpoints +.pytest_cache diff --git a/aicb/.omc/state/last-tool-error.json b/aicb/.omc/state/last-tool-error.json new file mode 100644 index 000000000..49943b7d1 --- /dev/null +++ b/aicb/.omc/state/last-tool-error.json @@ -0,0 +1,7 @@ +{ + "tool_name": "Bash", + "tool_input_preview": "{\"command\":\"head -22 aicb/results/workload/A5-GPT13B-world_size64-tp8-pp1-ep1-gbs1024-mbs1-seq8192-MOE-False-GEMM-False-flash_attn-True.txt | sed '2s/8334/20/' > /tmp/test_20l.txt && python3 -c \\\"\\nim...", + "error": "Exit code 1\nhead: aicb/results/workload/A5-GPT13B-world_size64-tp8-pp1-ep1-gbs1024-mbs1-seq8192-MOE-False-GEMM-False-flash_attn-True.txt: No such file or directory\nTraceback (most recent call last):\n File \"\", line 4, in \n with open('lld.json') as f: lld = json.load(f)\n ~~~~^^^^^^^^^^^^\nFileNotFoundError: [Errno 2] No such file or directory: 'lld.json'", + "timestamp": "2026-06-05T02:34:43.517Z", + "retry_count": 2 +} \ No newline at end of file diff --git a/aicb/Dockerfile b/aicb/Dockerfile new file mode 100644 index 000000000..9738ce710 --- /dev/null +++ b/aicb/Dockerfile @@ -0,0 +1,30 @@ +# Base image: Official NVIDIA PyTorch image with Python 3 and GPU support. +FROM nvcr.io/nvidia/pytorch:25.05-py3 + +# Install git for version control operations and clean up apt cache. +RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* + +# Set the application's working directory. +WORKDIR /workspace/AICB + +# [Optional] Configure pip and uv to use Aliyun mirror for faster package downloads. +RUN pip config set global.index-url http://mirrors.aliyun.com/pypi/simple +RUN pip config set install.trusted-host mirrors.aliyun.com +ENV UV_DEFAULT_INDEX="https://mirrors.aliyun.com/pypi/simple" + +RUN pip install --no-cache-dir uv + +# Copy only the requirements file first to leverage Docker's layer cache. +# This layer is rebuilt only when requirements.txt changes. +COPY requirements.txt . + +# Install Python dependencies using uv. +RUN UV_TORCH_BACKEND=auto uv pip install -v --system --no-cache-dir --no-build-isolation --break-system-packages -r requirements.txt + +# Copy the rest of the application source code into the image. +COPY . . + +RUN mv ./workload_generator /usr/local/lib/python3.12/dist-packages &&\ + mv ./utils /usr/local/lib/python3.12/dist-packages &&\ + mv ./log_analyzer /usr/local/lib/python3.12/dist-packages +ENV PATH="/workspace/AICB/.venv/bin:$PATH" \ No newline at end of file diff --git a/aicb/License b/aicb/License new file mode 100644 index 000000000..8fe0dbfdf --- /dev/null +++ b/aicb/License @@ -0,0 +1,14 @@ +/* +*Copyright (c) 2021, Alibaba Group; +*Licensed under the Apache License, Version 2.0 (the "License"); +*you may not use this file except in compliance with the License. +*You may obtain a copy of the License at + +* http://www.apache.org/licenses/LICENSE-2.0 + +*Unless required by applicable law or agreed to in writing, software +*distributed under the License is distributed on an "AS IS" BASIS, +*WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +*See the License for the specific language governing permissions and +*limitations under the License. +*/ \ No newline at end of file diff --git a/aicb/README.md b/aicb/README.md new file mode 100755 index 000000000..22c17c2a9 --- /dev/null +++ b/aicb/README.md @@ -0,0 +1,428 @@ +# Access AICB + +You can access the full suite of **SimAI** tools on **GitHub** via [**SimAI@github**](https://github.com/aliyun/SimAI) + +You can access AICB on **GitHub** via [**AICB@github**](https://github.com/aliyun/aicb) + +You can also access AICB on **Gitee** via [**AICB@gitee**](https://gitee.com/ali-ais-hpn/aicb) + +Welcome to join the SimAI community chat groups, with the DingTalk group on the left and the WeChat group on the right. + +
+ SimAI DingTalk + SimAI WeChat +
+ +
+ +# Latest News + +**Announcing AICB 2.1 (November 2025)** + +This major release introduces significant new capabilities, focusing on inference workloads and structural improvements. + +**✨ New Features** + +1. **Inference Workload Generation:** Added support for generating **inference** workloads for SimAI, initially for the DeepSeek, Qwen3-MoE and Qwen3-Next models. The DeepSeek support was advanced by [@Yan824](https://github.com/Yan824) and leverages the work from the [DeepSeek_Simulator](https://github.com/shenh10/DeepSeek_Simulator + ) project. The Qwen3-Moe and Qwen3-Next supports are based on [vLLM](https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/models/qwen3_moe.py). +2. **Expanded Training Support:** Training workload generation now includes support for DeepSeek. A special thanks to [@parthpower](https://github.com/parthpower) for contributing this feature! +3. **Megatron MoE Update:** Upgraded the Megatron integration to use its latest MoE communication method. + +**🛠️ Refactoring** + +1. **Improved Code Structure:** The `workload_generator/mocked_model` directory has been restructured into separate `inference` and `training` sub-modules for better organization. The existing training logic remains unchanged within the new `training` module. + +**⚠️ Notice** + +1. A Hopper-architecture GPU is required to profile computation time with AIOB for DeepSeek and Qwen3 inference models. This requirement stems from their use of specialized libraries, like [DeepGEMM](https://github.com/deepseek-ai/DeepGEMM). + +# Table of Contents + +- [Access AICB](#access-aicb) +- [Latest News](#latest-news) +- [Table of Contents](#table-of-contents) +- [AICB Overview](#aicb-overview) + - [Introduction](#introduction) + - [The benchmark suite in AICB](#the-benchmark-suite-in-aicb) +- [Setup](#setup) +- [Usage](#usage) + - [Running on physical GPU clusters](#running-on-physical-gpu-clusters) + - [Basic parameters that you need to set](#basic-parameters-that-you-need-to-set) + - [Running the whole benchmark suite](#running-the-whole-benchmark-suite) + - [Running workloads for Megatron](#running-workloads-for-megatron) + - [Running workloads for MOE](#running-workloads-for-moe) + - [Running workloads for DeepSpeed](#running-workloads-for-deepspeed) + - [Running workloads for DeepSeek](#running-workloads-for-deepseek) + - [Embedding the compuation patterns in the workload](#embedding-the-compuation-patterns-in-the-workload) + - [Generate Workloads for Training Simulation (SimAI)](#generate-workloads-for-training-simulation-simai) + - [Generating the workload description files for the whole benchmark suite](#generating-the-workload-description-files-for-the-whole-benchmark-suite) + - [Generating the workload description files for Megatron](#generating-the-workload-description-files-for-megatron) + - [Generating the workload description files for Moe](#generating-the-workload-description-files-for-moe) + - [Generating the workload description files for DeepSeek](#generating-the-workload-description-files-for-deepseek) + - [Generating the workload description files for DeepSpeed](#generating-the-workload-description-files-for-deepspeed) + - [Generate Workloads for Inference Simulation (SimAI)](#generate-workloads-for-inference-simulation-simai) + - [Running AICB with customized parameters](#running-aicb-with-customized-parameters) + - [Running customized workloads on physical GPU clusters](#running-customized-workloads-on-physical-gpu-clusters) + - [Generating customized workload description files](#generating-customized-workload-description-files) + - [Result Visualization](#result-visualization) +- [Tutorial](#tutorial) +- [Projects using AICB](#projects-using-aicb) + +# AICB Overview + +## Introduction + +AICB (Artificial Intelligence Communication Benchmark), is a novel benchmark suite for evaluating the communication system of a realistic and emulated GPU cluster from the pespectives of the emerging training and inference applications. Different from exisiting network benchmarks, AICB is designed to produce the communication workloads with precise patterns that are aligned to real-world applications. Taking the Large Language Model (LLM) training as an example, the workloads vary with the complicated combinations of models, parallel frameworks, and parameters of the models, parallel frameworks, and the collective communication libraries. In general, the scenarios suitable for using AICB include but not limited to 1) benchmarking and tuning of the communication system of a GPU cluster, 2) investigating and analyzing the communication patterns of specific application settings, 3) tools, e.g. simulators, that need workloads which are well described. + +## The benchmark suite in AICB + +There are a lot of parameters that influence the communication and computation patterns, which are (1) model parameters (e.g., hidden_size, num_layers, seq_len, etc.) and (2) framework parameters (e.g., world size, parallelization strategies (TP, PP, DP, SP), zero level, reduce_bucket_size/allgather_bucket_size, etc.). +For the sake of generality, we cover those typical settings using a smallest set of benchmarks rather than traversing all the combinations. To this end, we propose the benchmark suite as listed in the following table. +**Users can directly run all the selected workloads selected in AICB, or run part of the workloads, or even generate their own workloads.** +For more detailed information, please refer to [AICB_workload spec v1.1](workload/Workload_spec_v1.1.csv). +| id | Name | Sequence_length | Framework | TP | DP | PP | SP | Expert parallel number | Expert num | Zero_level | +|:---:|:-------------:|:---------------:|:---------:|:---:|:---------------------:|:---:|:------:|:----------------------:|:----------:|:----------:| +| 1 | LLaMA_7B | 2048 | Megatron | 1 | world_size/(PP*TP) | 1 | - | - | - | - | +| 2 | GPT_13B | 2048 | Megatron | 2 | world_size/(PP*TP) | 1 | enable | - | - | - | +| 3 | GPT_22B | 2048 | Megatron | 4 | world_size/(PP*TP) | 1 | - | - | - | - | +| 4 | LLaMA_65B | 4096 | Megatron | 8 | world_size/(PP*TP) | 2 | enable | - | - | - | +| 5 | GPT_175B | 2048 | Megatron | 8 | world_size/(PP*TP) | 8 | enable | - | - | - | +| 6 | GPT_175B | 2048 | Megatron | 8 | world_size/(PP*TP) | 8 | disable| - | - | - | +| 7 | Llama3_405B | 8192 | Megatron | 8 | world_size/(PP*TP) | 16 | enable | - | - | - | +| 8 | LLaMA_7B | 4096 | Deepspeed | 1 | world_size | 1 | - | - | - | 2 | +| 9 | LLaMA_65B | 4096 | Deepspeed | 1 | world_size | 1 | - | - | - | 3 | +| 10 | Mistral_8*7B | 2048 | Megatron | 2 | world_size/(PP*TP) | 1 | enable | 8 | 8 | - | + +# Setup + +You can follow the instrucitons below to quickly set up the environtments and run AICB. + +1. Installation from source code + + a. To initiate actual communication tasks, ensure that the runtime environment has all necessary dependencies, such as CUDA and [PyTorch](https://pytorch.org), already installed. For specific usage examples, see [Physical Execution](#physical-execution) + + b. To generate workload traffic patterns for large model parallel framework training, you can use a CPU-only environment. For specific usage examples, see [Generate Workloads ](#generate-workloads ) + +2. Installation from deb package (for Ubuntu systems) + + Currently, you can install the deb package on an NV-built NGC container [NGC's PyTorch container](https://ngc.nvidia.com/catalog/containers/nvidia:pytorch) to start running AICB. + + ```bash + docker pull nvcr.io/nvidia/pytorch:xx.xx-py3 + docker run --gpus all -it --rm -v /path/to/AICBench:/workspace/AICBench nvcr.io/nvidia/pytorch:xx.xx-py3 + dpkg -i /download/AICB_v1.0.deb + sh megatron_workload_with_aiob.sh -m 7 + ``` + +3. Composing a Docker image from Dockfile + + You can launch an instance of the Docker container with Dockerfile for quick start: + + ```bash + docker build -t image:latest . + docker run --gpus all -it --rm image:latest + ``` + +**Note**: The inference environment installed by this Dockerfile requires NVIDIA Hopper or Blackwell GPUs. If you are building on a Hopper (SM90) machine, add the following line to disable SM100-specific features: `ENV FLASH_MLA_DISABLE_SM100=1`. + +# Usage + +After installation, we provide three main usage scenarios for AICB: + +1. [Running on physical GPU clusters](#running-on-physical-gpu-clusters) +2. [Generating workload descrption files for simulation](#generate-workloads-for-simulation-simai) +3. [Customized parameters](#customized-parameters). + +There is a tutorial including all the details, please refer to [the tutorial](training/tutorial.md). + +## Running on physical GPU clusters + +For running AICB on a physical machine, we provide both [scripts](scripts/megatron_gpt.sh) for quick start and [methods](aicb.py) for executing custom cases. + +### Basic parameters that you need to set + +When running on a physical machine, additional configuration of environment variables required by PyTorch is necessary. + +```bash +--nnodes Number of nodes: $WORLD_SIZE +--node_rank Rank of the node: $RANK +--nproc_per_node Number of GPUs per node: $NUM_GPUS +--master_addr Master address: $MASTER_ADDR +--master_port Master port: $MASTER_PORT +``` + +### Running the whole benchmark suite + +You can directly execute all the test cases provided in our AICB workload specification v1.0 in physical GPU cluster by utilizing the [run_suites](run_suites.py) script. This script ensures that all parallel framworks are covered, allowing you to validate and analyze the performance and behavior of various workloads efficiently. + +### Running workloads for Megatron + +For the `Megatron parallel framework`, you can quickly start using the scripts/megatron_gpt.sh script file. + +```bash +sh scripts/megatron_gpt.sh \ +--nnodes 1 --node_rank 0 --nproc_per_node 8 --master_addr localhost --master_port 29500 \ +-m 7 --world_size 8 --tensor_model_parallel_size 2 --pipeline_model_parallel 1 \ +--frame Megatron --global_batch 16 \ +--micro_batch 1 --seq_length 2048 --swiglu --use_flash_attn --aiob_enable +``` + +### Running workloads for MOE + +For `Moe` , you can quickly start it using the [scripts/megatron_gpt.sh](scripts/megatron_gpt.sh) script file. + +```bash +sh scripts/megatron_gpt.sh \ +--nnodes 1 --node_rank 0 --nproc_per_node 8 --master_addr localhost --master_port 29500 \ +-m moe --world_size 8 --tensor_model_parallel_size 4 --pipeline_model_parallel 1 \ +--moe_enable --expert_model_parallel_size 1 \ +--frame Megatron --global_batch 16 \ +--num_experts 4 --moe_router_topk 2 \ +--micro_batch 1 --sp --grouped_gemm --aiob_enable --swiglu --use_flash_attn +``` + +### Running workloads for DeepSpeed + +For the `DeepSpeed` parallel framework, you can quickly start it using the [scripts/deepspeed_llama.sh](scripts/deepspeed_llama.sh) script file. Currently, the DeepSpeed framework does not support `--aiob_enable` or `--comp_filepath`, but you can choose to use fixed computation times (please refer to [the tutorial](training/tutorial.md)). + +```bash +sh scripts/deepspeed_llama.sh \ +--zero_stage 3 -m 65 --epoch_num 100 \ +--reduce_bucket_size=1000000000 --allgather_bucket_size=500000000 \ +--param_persistence_threshold=1000000 \ +``` + +### Running workloads for DeepSeek + +For `DeepSeek`, one can use the [scripts/megatron_gpt.sh](./scripts/megatron_gpt.sh) with `--frame=DeepSeek` + +```bash +sh scripts/megatron_gpt.sh \ + --frame DeepSeek \ + --tensor_model_parallel_size 4 \ + --pipeline_model_parallel 1 \ + --moe_enable \ + --expert_model_parallel_size 1 \ + --global_batch 4 \ + -m deepseek \ + --num_experts 4 \ + --micro_batch 1 \ + --sp --swiglu --world_size 4 --num_layers 10 --aiob_enable +``` + +### Embedding the compuation patterns in the workload + +To mirror the real-world workloads with both computation and communicaiton, we developed a sub-module, AIOB, that is used to generate computation patterns. +In AICB, we can enable AIOB to embed the computation time into the workloads. + +For the Megatron parallel framework, the `--aiob_enable` option allows for capturing the computation time of each operation in the actual model. +If we do not set `--aiob_enable`, only fixed computation times can be applied. (Please refer to [the tutorial](training/tutorial.md)) + +* Running workloads with computation times generated by AIOB. After running, we can get an extra computation desrcription file describing the computation times for the main computation kernels in the directory of [results/aiob_outputs](results/aiob_outputs). + Note that the computation times are obtained through the execution of computation kernels on the specific GPU. + The following commands does not generate the computation descrition file, but also run the workload in the real GPU cluster. + + ```bash + sh scripts/megatron_gpt.sh \ + -m 7 --world_size 8 --tensor_model_parallel_size 2 --pipeline_model_parallel 1 \ + --frame Megatron --global_batch 16 \ + --micro_batch 1 --seq_length 2048 \ + --swiglu --use_flash_attn --aiob_enable + ``` + +* Running workload with computation time through an existing computation decription file. + Users can defined their own computation times or directly use the files we provided. + By specifying the computation description file with the `--comp_filepath` option, you can embed computation times before running the workload on a physical machine. + + ```bash + sh scripts/megatron_gpt.sh \ + -m 7 --world_size 8 --tensor_model_parallel_size 2 --pipeline_model_parallel 1 \ + --frame Megatron --global_batch 16 --micro_batch 1 \ + --seq_length 2048 --swiglu --use_flash_attn \ + --aiob_enable \ + --comp_filepath workload/aiob_inputs/Example.txt + ``` + + ## Generate Workloads for Training Simulation (SimAI) + + In addition to running the AICB in the GPU clusters, AICB also generates the workload description files which can be used for simulation or further analysis. + In this release, we provide [scripts](scripts/megatron_workload_with_aiob.sh) for quickly generating workloads for SimAI. + For now Linear Attention Models are not supported. To customize your own workloads, please refer to [Customized parameters](workload_generator/mocked_model/MockedModel.py) and add new models. + +### Generating the workload description files for the whole benchmark suite + +You can generate all the workload description files with [generate_suite]() as specified in our AICB workload spec v1.0. Once these files are created, you can execute them using the SimAI to test and analyze various scenarios. + +### Generating the workload description files for Megatron + +Here, you can use the script [scripts/megatron_workload.sh](scripts/megatron_workload_with_aiob.sh) and the parameter `--model_size` (7/13/22/175/moe) to generate the corresponding workload description file. For the computation part of the model, you can choose to enable AIOB by using the `--aiob_enable` option. If AIOB is not used, the Workload will be filled with a fixed computation time by default. + +* Generating workload description files with computation times generated by AIOB. + + ```bash + sh ./scripts/megatron_workload_with_aiob.sh \ + -m 7 --world_size 4096 \ + --tensor_model_parallel_size 2 --pipeline_model_parallel 1 \ + --frame Megatron --global_batch 8192 \ + --micro_batch 1 --seq_length 4096 \ + --swiglu --use_flash_attn --aiob_enable + ``` + +* Generating workload description files with computation time through an existing computation decription file. + +```bash +sh ./scripts/megatron_workload_with_aiob.sh -m 7 \ +--world_size 4096 --tensor_model_parallel_size 2 --pipeline_model_parallel 1 \ +--frame Megatron --global_batch 8192 \ +--micro_batch 1 --seq_length 4096 --swiglu \ +--use_flash_attn --aiob_enable \ +--comp_filepath workload/aiob_inputs/Example.txt +``` + +### Generating the workload description files for Moe + +For the Moe, you can also use [scripts/megatron_workload_with_aiob.sh](scripts/workload_megatron.sh) to generate the corresponding model's workload file. + +```bash +sh scripts/megatron_workload_with_aiob.sh \ +-m moe --world_size 512 --tensor_model_parallel_size 2 --pipeline_model_parallel 1 --sp --ep 16 \ +--num_experts 64 --moe_router_topk 2 --moe_grouped_gemm --moe_enable \ +--frame Megatron --global_batch 1024 \ +--micro_batch 1 --seq_length 4096 --swiglu \ +--use_flash_attn --aiob_enable +``` + +### Generating the workload description files for DeepSeek + +For the DeepSeek, you can also use [scripts/megatron_workload_with_aiob.sh](scripts/megatron_workload_with_aiob.sh) + +```bash +sh scripts/megatron_workload_with_aiob.sh \ +-m deepseek16 --world_size 2048 --tensor_model_parallel_size 1 --pipeline_model_parallel 1 --sp --ep 16 \ +--moe_router_topk 6 --moe_enable \ +--frame DeepSeek --global_batch 4096 \ +--micro_batch 1 --seq_length 4096 --swiglu \ +--aiob_enable +``` + +### Generating the workload description files for DeepSpeed + +For the `DeepSpeed` parallel framework, you can use [scripts/workload_deepspeed.sh](scripts/workload_deepspeed.sh) to generate the corresponding workload description file. + +```bash +sh ./scripts/workload_deepspeed.sh -m 7 +``` + +## Generate Workloads for Inference Simulation (SimAI) + +Use the [scripts/inference_workload_with_aiob.sh](scripts/inference_workload_with_aiob.sh) script to generate workload description files for **DeepSeek**, **Qwen3-MoE**, and **Qwen3-Next**. Configuration is managed through a combination of a model-specific JSON file and command-line arguments. The JSON file defines the core model architecture (e.g., number of layers, hidden size), while command-line arguments are used to specify runtime parameters such as sequence length, batch size, and parallelism settings. + +> **Note:** +> +> 1. **Simulator Compatibility:** Currently, all inference workloads generated by AICB are compatible only with the SimAI-Analytical simulator, which models communication time using `autobusbw`. Support for SimAI to simulate DeepEP communication is under active development and will be included in a future release. +> 2. **Computation Modeling:** Inside the configuration file, you can set `aiob_enable`: true to enable realistic, hardware-based computation modeling with AIOB. If this option is set to false and the aiob output file is not found, a fixed computation time will be used by default. +> 3. **Experimental Support for Qwen3-Next:** The simulation for Qwen3-Next is an experimental feature. The underlying performance profiling data may not be fully accurate, especially for GDN. Contributions from the community to improve this feature are highly welcome! + +Here are some examples: + +- Generating DeepSeek workload description files using default parameters and enables AIOB for computation modeling, which will create a `decode` workload using default config fle and default settings. + +```bash +sh ./scripts/inference_workload_with_aiob.sh -m deepseek-671B -a +``` + +- Generating Qwen3Moe `prefill` workload description files with a specific config file, while setting seq_length=1024, micro_batch=4, world_size=ep_size=32. AIOB is not enabled in this case. + +```bash +sh ./scripts/inference_workload_with_aiob.sh -m qwen3-235B \ +-c ./scripts/inference_configs/qwen3_moe_default.json \ +-s 1024 -b 4 -w 32 -e 32 -p prefill +``` + +> For a complete list of all available options and their descriptions, run the script with the `--help` flag. + +> Use the [scripts/inference_configs/config_gen.py](scripts/inference_configs/config_gen.py) script to generate execution scripts based on different argument combinations. + +## Generate Workloads for Inference Simulation (Vidur) + +To generate a workload for Vidur-Alibaba, use the [workload_generator/Vidur_workload_generator.py](workload_generator/Vidur_workload_generator.py) script. It produces a CSV file detailing each layer's ID, name, computation time, and communication size. + +## Running AICB with customized parameters + +In addition to quick start, you can also customize the model parameters in detail to run on physical clusters or generate the required workloads for simulation and analysis. For more detailed parameter descriptions and more Example, please refer to [the tutorial](training/tutorial.md). + +### Running customized workloads on physical GPU clusters + +The current entry file for running custom cases is [aicb.py](aicb.py). By using this file, you can flexibly choose more parameters for tuning. + +```bash +# Megatron Example +torchrun \ +--nnodes $WORLD_SIZE \ +--node_rank $RANK \ +--nproc_per_node gpu \ +--master_addr $MASTER_ADDR \ +--master_port $MASTER_PORT \ +./aicb.py --frame=Megatron --world_size=$((WORLD_SIZE*8)) --tensor_model_parallel_size=$tensor_model_parallel_size \ + --micro_batch=$batch_size --global_batch=$((WORLD_SIZE*8*batch_size/tensor_model_parallel_size)) --epoch_num=$epoch_num \ + --num_layers=$num_layers --hidden_size=$hidden_size --ffn_hidden_size=$ffn_hidden_size --num_attention_heads=$num_attention_heads \ + $sp_enable --seq_len=$seq_len --vocab_size=$vocab_size --aiob_enable=$enable +``` + +### Generating customized workload description files + +Similarly, when generating workloads, you can also customize the model training parameters and modifying the generated files to generate your own workload file for simulation. This can be achieved by using the following files: +[generate custom description file](workload_generator/SimAI_training_workload_generator.py) + +Here is an example: + +```bash +python -m workload_generator.SimAI_training_workload_generator \ +--world_size=32 --global_batch=64 --micro_batch=1 \ +--num_layers=8 --num_attention_heads=176 --hidden_size=5120 \ +--tensor_model_parallel_size=2 --seq_length=4096 --swiglu --ffn_hidden_size=16384 \ +--moe_router_topk=4 --enable_sequence_parallel --expert_model_parallel_size=16 \ +--num_experts=64 --moe_grouped_gemm --moe_enable --num_experts=4 +``` + +## Result Visualization + +This section provides an introduction to the result visualization feature. + +Supported Formats: The `.csv` files is supported for visualization, including results from physical cluster runs and workload files. + +Usage: +Both Post-Run and generated workload files could be visualized. You can use the [visualize_script](visualize/generate.py) to visualize the results. +Here is an example of a workload file: + +```bash +python -m visualize.generate ./local_megatron_workload.csv only_workload +``` + +Post-Run results visualization examples: + +```bash +python -m visualize.generate ./megatron_postrun.csv +``` + +The output results are located in the `results/visual_output` directory. You can view the output style through the `example.html` file located in this directory. The generated visualization file is an HTML file, which can be opened and viewed in a web browser, just like this. +![Scaling Graph](./images/readme_01.png) + +The results consist of several parts: + +- Communication Results Pie Chart: Shows the quantity and proportion of various collective communications under the given training hyperparameters. +- Communication Type Scatter Plot: Displays the message size and communication count for each type of communication under the given training hyperparameters. For results from actual physical cluster runs, it will also show the corresponding BusBw. +- CDF of Message Sizes in Collective Communications: Illustrates the distribution of message sizes across different types of collective communications. +- Comm Group Scatter Plot: Shows the message size and communication count for different model training communication groups. For results from actual physical cluster runs, it will also show the corresponding BusBw. +- Computation and Communication Timeline (Supported for Physical Cluster Runs Only): Displays the timeline of computation and communication events during AICB runs. The timeline can be dragged to observe specific computation and communication events. +- Overall Computation and Communication Proportion (Supported for Physical Cluster Runs Only): Shows the proportion of total time spent on computation and communication during AICB runs. + +# Tutorial + +We provide a tutorial for users to quickly get started with AICB. [the tutorial](./training/tutorial.md) + +# Projects using AICB + +Below are some of the projects where we have directly used AICB: + +* AICB is part of the SimAI project which is led by Alibaba Cloud. Researchers who use AICB can cite our paper "SimAI: Unifying Architecture Design and Performance Tunning for Large-Scale Large Language Model Training with Scalability and Precision" (NSDI’25). diff --git a/aicb/aicb.py b/aicb/aicb.py new file mode 100755 index 000000000..43ca31d5c --- /dev/null +++ b/aicb/aicb.py @@ -0,0 +1,94 @@ +""" +Copyright (c) 2021, Alibaba Group; +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import torch +from utils.utils import get_args, get_comp_out, extract_averages, Comp_with_aiob +from utils.benchmark_logger import bench_logger +from workload_generator.mocked_model.training.MockedDeepspeed import DeepspeedForCausalLM +from workload_generator.mocked_model.training.MockedDeepSeek import DeepSeekV3Model +from workload_generator.mocked_model.training.MockedMegatron import MegatronModel +from workload_generator.generate_deepspeed_stage1_2_workload import ( + DeepSpeedStage1, + DeepSpeedStage2, +) +from workload_generator.generate_deepspeed_stage3_workload import DeepSpeedStage3 +from workload_generator.generate_megatron_workload import MegatronWorkload +from workload_generator.generate_collective_test import Collective_Test +from workload_applyer import WorkloadApplyer +from utils.utils import * + +if __name__ == "__main__": + args = get_args() + if not hasattr(args, "backend"): + args.backend = "nccl" + torch.distributed.init_process_group(backend=args.backend) + args.world_size = torch.distributed.get_world_size() + args.rank = torch.distributed.get_rank() + if args.frame == "Megatron": + model = MegatronModel(args) + workload_generator = MegatronWorkload(args, model) + elif args.frame == "DeepSpeed": + model = DeepspeedForCausalLM(args) + if args.stage == 1: + workload_generator = DeepSpeedStage1(args, model) + elif args.stage == 2: + workload_generator = DeepSpeedStage2(args, model) + elif args.stage == 3: + workload_generator = DeepSpeedStage3(args, model) + elif args.frame == "collective_test": + workload_generator = Collective_Test(args, None) + elif args.frame == "DeepSeek": + model = DeepSeekV3Model(args) + workload_generator = MegatronWorkload(args, model) + workload = workload_generator() + if args.aiob_enable and (args.frame == "Megatron" or args.frame == "DeepSeek"): + + params = model.parameters() + args.model_param = sum(p.numel() for p in params) + args.activation_memory = 0 + for sub_module in model.child_modules(): + if hasattr(sub_module, "activation_memory"): + args.activation_memory += sub_module.activation_memory() + print("model_param:", args.model_param) + if args.comp_filepath == None: + local_rank = torch.distributed.get_rank() % torch.cuda.device_count() + if local_rank == 0: + filepath = get_comp_out(args) + else: + filepath = get_aiob_path(args) + torch.distributed.barrier() + compute_cache = extract_averages(filepath,args) + else: + print("comp_filepath:", args.comp_filepath) + compute_cache = extract_averages(args.comp_filepath,args) + workload = Comp_with_aiob(workload, compute_cache) + if torch.distributed.get_rank() == 0: + filename = f"{workload_generator.name}_{args.model_name}_sp_{args.enable_sequence_parallel}_iteration_{args.epoch_num}_computationEnable_{args.computation_enable}_{args.world_size}n.csv" + workload.dump(filename) + if not args.workload_only : + applyer = WorkloadApplyer(workload=workload, args=args) + cpu_time = applyer.apply_workload() + if torch.distributed.get_rank() == 0: + bench_logger.analyze_comm_log() + if args.frame != "collective_test": + bench_logger.analyze_comm_time() + csv_filename = bench_logger.dump_log(filename) + if args.enable_visual: + try: + from visualize.generate import visualize_output + visualize_output(csv_filename,False) + except ImportError: + print("visualize_output is not available because required library is not found") + + print( + f"total time for {args.frame} and {args.epoch_num} iterations is {cpu_time:.4f} s" + ) diff --git a/aicb/core/__init__.py b/aicb/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/aicb/core/grouped_gemm_util.py b/aicb/core/grouped_gemm_util.py new file mode 100644 index 000000000..e7ef79d79 --- /dev/null +++ b/aicb/core/grouped_gemm_util.py @@ -0,0 +1,20 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. + +try: + import grouped_gemm +except ImportError: + grouped_gemm = None + + +def grouped_gemm_is_available(): + return grouped_gemm is not None + + +def assert_grouped_gemm_is_available(): + assert grouped_gemm_is_available(), ( + "Grouped GEMM is not available. Please run " + "`pip install git+https://github.com/fanshiqing/grouped_gemm@v1.0`." + ) + + +ops = grouped_gemm.ops if grouped_gemm_is_available() else None diff --git a/aicb/download/AICB_v1.0.deb b/aicb/download/AICB_v1.0.deb new file mode 100644 index 000000000..46a913db9 Binary files /dev/null and b/aicb/download/AICB_v1.0.deb differ diff --git a/aicb/images/detail_log.png b/aicb/images/detail_log.png new file mode 100644 index 000000000..7a47a2cce Binary files /dev/null and b/aicb/images/detail_log.png differ diff --git a/aicb/images/readme_01.png b/aicb/images/readme_01.png new file mode 100644 index 000000000..7fa49a50d Binary files /dev/null and b/aicb/images/readme_01.png differ diff --git a/aicb/images/result_log.png b/aicb/images/result_log.png new file mode 100644 index 000000000..f7278d0b8 Binary files /dev/null and b/aicb/images/result_log.png differ diff --git a/aicb/images/simai_dingtalk.jpg b/aicb/images/simai_dingtalk.jpg new file mode 100644 index 000000000..ee2d04164 Binary files /dev/null and b/aicb/images/simai_dingtalk.jpg differ diff --git a/aicb/images/simai_wechat.jpeg b/aicb/images/simai_wechat.jpeg new file mode 100644 index 000000000..e70c9f02b Binary files /dev/null and b/aicb/images/simai_wechat.jpeg differ diff --git a/aicb/images/time_log.png b/aicb/images/time_log.png new file mode 100644 index 000000000..3ec14ad86 Binary files /dev/null and b/aicb/images/time_log.png differ diff --git a/aicb/images/tutorial_1.png b/aicb/images/tutorial_1.png new file mode 100644 index 000000000..3eed412eb Binary files /dev/null and b/aicb/images/tutorial_1.png differ diff --git a/aicb/images/tutorial_2.png b/aicb/images/tutorial_2.png new file mode 100644 index 000000000..d66248373 Binary files /dev/null and b/aicb/images/tutorial_2.png differ diff --git a/aicb/images/tutorial_3.png b/aicb/images/tutorial_3.png new file mode 100644 index 000000000..f97181043 Binary files /dev/null and b/aicb/images/tutorial_3.png differ diff --git a/aicb/images/tutorial_4.png b/aicb/images/tutorial_4.png new file mode 100644 index 000000000..24ba0d08b Binary files /dev/null and b/aicb/images/tutorial_4.png differ diff --git a/aicb/images/tutorial_5.png b/aicb/images/tutorial_5.png new file mode 100644 index 000000000..652aa4b2c Binary files /dev/null and b/aicb/images/tutorial_5.png differ diff --git a/aicb/images/tutorial_6.png b/aicb/images/tutorial_6.png new file mode 100644 index 000000000..35113088c Binary files /dev/null and b/aicb/images/tutorial_6.png differ diff --git a/aicb/images/tutorial_7.png b/aicb/images/tutorial_7.png new file mode 100644 index 000000000..89d16de9f Binary files /dev/null and b/aicb/images/tutorial_7.png differ diff --git a/aicb/log_analyzer/__init__.py b/aicb/log_analyzer/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/aicb/log_analyzer/analyze_res_csv.py b/aicb/log_analyzer/analyze_res_csv.py new file mode 100644 index 000000000..b5a652435 --- /dev/null +++ b/aicb/log_analyzer/analyze_res_csv.py @@ -0,0 +1,38 @@ +import pandas as pd +from log_analyzer.utils import convert_msg_to_size, convert_size_to_msg +import sys + +def analyze_csv(file_path): + df = pd.read_csv(file_path) + + df = df.dropna(subset=['busbw']) + + df['busbw'] = pd.to_numeric(df['busbw'], errors='coerce') + + df = df.dropna(subset=['busbw']) + + def exclude_min(group): + if len(group) > 1: + group = group.sort_values(by='busbw') + return group.iloc[2:] + return group + + df_excluded_min = df.groupby(['comm_type', 'comm_group', 'msg_size']).apply(exclude_min).reset_index(drop=True) + grouped = df_excluded_min.groupby(['comm_type', 'comm_group', 'msg_size']).agg( + busbw_mean=('busbw', 'mean'), + busbw_max=('busbw', 'max'), + busbw_min=('busbw', 'min'), + busbw_std=('busbw', 'std'), + occurrence_count=('busbw', 'size') + ).reset_index() + grouped['msg_size'] = grouped['msg_size'].apply(convert_size_to_msg) + return grouped + + +if __name__ == '__main__': + if len(sys.argv) < 2: + print("Usage: python -m log_analyzer.analyze_res_csv ") + sys.exit(1) + grouped = analyze_csv(sys.argv[1]) + print(grouped) + diff --git a/aicb/log_analyzer/ds_comm_log_analyzer.py b/aicb/log_analyzer/ds_comm_log_analyzer.py new file mode 100644 index 000000000..3492a412e --- /dev/null +++ b/aicb/log_analyzer/ds_comm_log_analyzer.py @@ -0,0 +1,134 @@ +""" +Copyright (c) 2021, Alibaba Group; +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# /usr/bin/python3 +from utils.utils import CommType, CommGroup +from log_analyzer.utils import convert_msg_to_size, convert_size_to_msg +from utils.benchmark_logger import BenchLogger +from log_analyzer.log import LogItem, Log + +COMM_OP = "comm op" +CALLER_FUNC = "Caller Func" +TIME_MS = "time (ms)" +MSG_SIZE = "msg size" +LOG_STARTER = "[rank 0]" +WORLD_SIZE = 16 +TP_SIZE = 4 +DP_SIZE = 4 +# LOG_STARTER = "[INFO] " + + +def clean_s(s): + return s.strip("[]\n\t ") + + +def string2comm_type(s): + if "all_gather" in s: + return CommType.all_gather + if "reduce_scatter" in s: + return CommType.reduce_scatter + if "all_reduce" in s: + return CommType.all_reduce + if "broadcast" in s: + return CommType.broadcast + if "barrier" in s: + return CommType.barrier + if "reduce" in s: + return CommType.reduce + print(f"WARNING cannot convert {s} to CommType") + return CommType.epoch_end + + +def parse_ds_log_item(line): + index = line.lower().find(LOG_STARTER) + if index == -1: + return None + item_list = line[index + len(LOG_STARTER) :].split("|") + item = {} + for raw_item in item_list: + if "epoch" in raw_item: + split_text = raw_item.split() + numbers = [word for word in split_text if word.isdigit()] + item["epoch_num"] = int(numbers[0]) + continue + if "micro_step" in raw_item: + split_text = raw_item.split() + numbers = [word for word in split_text if word.replace(".", "").isdigit()] + item["iter_time"] = float(numbers[0]) + continue + if ":" not in raw_item: + continue + key, value = raw_item.split(":") + key, value = clean_s(key), clean_s(value) + if key == COMM_OP: + item["comm_type"] = string2comm_type(value) + elif key == MSG_SIZE or MSG_SIZE in key: + item["msg_size"] = convert_msg_to_size(value) + elif key == CALLER_FUNC: + item["stage"] = value + elif key == TIME_MS or TIME_MS in key: + item["elapsed_time"] = float(value) + if key == "group": + group = eval(value) + if len(group) == WORLD_SIZE: + item["group"] = CommGroup.all + elif len(group) == TP_SIZE: + item["group"] = CommGroup.tp_group + elif len(group) == DP_SIZE: + item["group"] = CommGroup.dp_group + elif "algbw" in key: + item["algbw"] = float(value) + elif "busbw" in key: + item["busbw"] = float(value) + else: + try: + item[key] = float(value) + except: + item[key] = value + return item + + +def parse_ds_comm_log(filename): + comm_log = Log() + with open(filename, "r") as f: + lines = f.read().split("\n") + for line in lines: + if "After initializing ZeRO optimizer" in line: + comm_log.add_comm_log(LogItem(comm_type=CommType.epoch_end)) + continue + elif "microstep" in line: + comm_log.add_comm_log(LogItem(comm_type=CommType.epoch_end)) + continue + log = parse_ds_log_item(line) + if log is None: + continue + if "comm_type" in log: + log_item = LogItem( + comm_type=log["comm_type"], + comm_group=log.get("group", CommGroup.dp_group), + msg_size=log["msg_size"], + ) + log_item._elapsed_time = log.get("elapsed_time", -1) + log_item.algbw, log_item.busbw = log.get("algbw", -1), log.get( + "busbw", -1 + ) + comm_log.add_comm_log(log_item) + return comm_log + + +if __name__ == "__main__": + import sys + + filename = sys.argv[1] + comm_log = parse_ds_comm_log(filename) + comm_log.analyze() diff --git a/aicb/log_analyzer/log.py b/aicb/log_analyzer/log.py new file mode 100755 index 000000000..c1ef1610a --- /dev/null +++ b/aicb/log_analyzer/log.py @@ -0,0 +1,355 @@ +""" +Copyright (c) 2021, Alibaba Group; +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os,math +import pickle +import csv +import dataclasses +import logging +import numpy as np +from typing import Union, Dict, List +from utils.utils import CommType, CommGroup +from log_analyzer.utils import convert_size_to_msg, calc_bw_log +import copy + +logger = logging.getLogger(__name__) + +@dataclasses.dataclass +class LogItem: + comm_type: CommType = dataclasses.field(default=None) + comm_group: CommGroup = dataclasses.field(default=None) + comm_group_size: int = dataclasses.field(default=None) + msg_size: float = dataclasses.field(default=0) + + stage: str = dataclasses.field(default="") + dst: int = dataclasses.field(default=None) + src: int = dataclasses.field(default=None) + additional: str = dataclasses.field(default="") + + _elapsed_time: float = dataclasses.field(default=None) + algbw: float = dataclasses.field(default=None) + busbw: float = dataclasses.field(default=None) + count: float = dataclasses.field(default=1) + ranks: list = dataclasses.field(default=None) + + @property + def elapsed_time(self) -> float: + return self._elapsed_time + + @elapsed_time.setter + def elapsed_time(self, elapsed_time): + self._elapsed_time = elapsed_time + self.algbw, self.busbw = calc_bw_log( + self.comm_type, self.msg_size, elapsed_time, self.comm_group_size + ) + + def is_epoch_end(self): + return self.comm_type == CommType.epoch_end + + def is_workload(self): + return self.elapsed_time is None + + def view_as_ds_log(self): + log_str = f"[RANK 0] comm op: {self.comm_type} | comm group: {self.comm_group}" + log_str += " | time (ms): {:.2f}".format(self.elapsed_time) + if self.comm_type == CommType.computation or self.additional == 'overlap': + log_str += " | msg size: " + '0' + log_str += " | algbw (GB): " + '0' + log_str += " | busbw (GB): " + '0' + else: + log_str += " | msg size: " + convert_size_to_msg(self.msg_size) + log_str += " | algbw (GB): {:.2f} ".format(self.algbw) + log_str += " | busbw (GB): {:.2f} ".format(self.busbw) + return log_str + + def csv_header(self): + return ",".join([k for k in self.__dict__.keys()]) + + def view_as_csv_line(self): + parts = [] + for k in self.__dict__.keys(): + v = getattr(self, k) + if k == 'ranks': + if v is not None: + parts.append('"' + ','.join(str(r) for r in v) + '"') + else: + parts.append('') + else: + parts.append(str(v)) + return ','.join(parts) + + def __str__(self): + if self.is_workload(): + return "None" + return "None" + + +def _print_stage_log(stage_name: str, stage_count: int, comm_type_info: Dict, primary_key: List[str], agg_key: List[str], performance_key: List[str], busbw_key: List[str]): + header = f"{'Comm_Type':<15} {'Comm_Group':<12} {'Message_Size':<12} {'Count':<12} {'Avg_Elapsed_Time ± Std ':<24} {'Avg_BusBw ± Std':<24}\n" + separator = "-" * len(header) + "\n" + log_str = separator + header + separator + + for pkey in sorted(comm_type_info.keys()): + row_str = "" + values = {} + for i, pkey_name in enumerate(primary_key): + value = pkey[i] if pkey_name != "msg_size" else convert_size_to_msg(pkey[i]) + values[pkey_name] = value + for key in agg_key: + value = comm_type_info[pkey][key] + value = convert_size_to_msg(value) if key == "msg_size" else f"{value:.2f}" + values[key] = value + for key in performance_key: + performance_value_list = sorted(comm_type_info[pkey][key]) + values[f'avg_{key}'] = f"{np.mean(performance_value_list):.2f}±{np.std(performance_value_list):.2f}" + values[f'min_{key}'] = f"{performance_value_list[0]:.2f}" + values[f'max_{key}'] = f"{performance_value_list[-1]:.2f}" + + for key in busbw_key: + busbw_value_list = sorted(comm_type_info[pkey][key]) + values[f'avg_{key}'] = f"{np.mean(busbw_value_list):.2f}±{np.std(busbw_value_list):.2f}" + + row_str += f"{values['comm_type']:<15} {values['comm_group']:<12} {values['msg_size']:<12} {values['count']:<16} {values['avg__elapsed_time']:<24} {values['avg_busbw']:<18}\n" + log_str += row_str + + return log_str + + +def _analyze_stage_log(comm_log: List[Dict], stage: str, comm_info: Dict[str, Dict]): + def __update_info( + info_dict, + log, + primary_key: List[str], + agg_key: List[str], + performance_key: List[str], + busbw_key: List[str], + ): + primary_key = tuple(log[key] for key in primary_key) + if primary_key not in info_dict: + info_dict[primary_key] = dict((key, 0) for key in agg_key) + info_dict[primary_key].update(dict((key, []) for key in performance_key)) + info_dict[primary_key].update(dict((key, []) for key in busbw_key)) + for key in agg_key: + info_dict[primary_key][key] += log[key] + for key in performance_key: + info_dict[primary_key][key].append(log[key]) + for key in busbw_key: + info_dict[primary_key][key].append(log[key]) + + if stage not in comm_info: + comm_info[stage] = { + "count": 0, + "comm_type_info": {}, + "detailed_comm_type_info": {}, + } + comm_info[stage]["count"] += 1 + # key: comm_type, value: count, time_ms + comm_type_info = comm_info[stage]["comm_type_info"] + # key: comm_type, msg_size, value: count, time_ms + detailed_comm_type_info = comm_info[stage]["detailed_comm_type_info"] + for log in comm_log: + if log.comm_type != CommType.computation: + __update_info( + comm_type_info, + log.__dict__, + ["comm_type", "comm_group"], + ["count", "msg_size"], + ["_elapsed_time"], + ["busbw"], + ) + __update_info( + detailed_comm_type_info, + log.__dict__, + ["comm_type", "comm_group", "msg_size"], + ["count"], + ["_elapsed_time"], + ["busbw"], + ) + + +class Log: + def __init__(self) -> None: + self.comm_logs = [] + self.comm_log_each_epoch = [[]] + self.epoch_times = [] + + def add_comm_log(self, comm_log: LogItem): + if ( + comm_log.is_epoch_end() + and len(self.comm_logs) > 0 + and not self.comm_logs[-1].is_epoch_end() + ): + self.comm_logs.append(comm_log) + self.comm_log_each_epoch.append([]) + self.epoch_times.append(comm_log.elapsed_time) + return + self.comm_logs.append(comm_log) + self.comm_log_each_epoch[-1].append(comm_log) + + def analyze(self, print_fn=print): + comm_info: Dict[str, Dict] = {} + _analyze_stage_log(self.comm_log_each_epoch[0], "init", comm_info) + for e_log in self.comm_log_each_epoch[1:]: + _analyze_stage_log(e_log, "train", comm_info) + for stage in comm_info.keys(): + if stage != "init": + stage_count = comm_info[stage]["count"] + comm_type_info = comm_info[stage]["comm_type_info"] + detailed_comm_type_info = comm_info[stage]["detailed_comm_type_info"] + + log_str = _print_stage_log(stage, stage_count, detailed_comm_type_info, ["comm_type", "comm_group", "msg_size"], ["count"], ["_elapsed_time"], ["busbw"]) + print_fn(f"\n\tDetailed comm info for AICB {stage} stage\n{log_str}") + return comm_info + + def dump(self, filename): + default_comm_folder_path = "results/comm_logs/" + if not os.path.exists(default_comm_folder_path): + os.makedirs(default_comm_folder_path, exist_ok=True) + if "." in filename: + filename = filename.split(".")[0] + filename = os.path.join("results/comm_logs/", filename) + csv_filename = filename + "_log.csv" + with open(csv_filename, "w") as f: + f.write(self.comm_logs[0].csv_header() + "\n") + for log_item in self.comm_logs: + log_item_write = copy.deepcopy(log_item) + if(log_item_write.comm_type == CommType.computation): + msg_size_str = "("+' '.join(str(shape).replace(',', '') for shape in log_item_write.msg_size)+")" + log_item_write.msg_size = msg_size_str + f.write(log_item_write.view_as_csv_line() + "\n") + del log_item_write + return csv_filename + + @staticmethod + def load(filename): + filename = filename.split(".") + filename[-1] = "pkl" + filename = ".".join(filename) + return pickle.load(open(filename, "rb")) + + def _get_elapsed_time(self): + return self.epoch_times + + def analyze_time(self, print_fn=print): + self.epoch_times.pop(0) + max_val = max(self.epoch_times) + min_val = min(self.epoch_times) + mean_val = sum(self.epoch_times) / len(self.epoch_times) + + variance = sum((x - mean_val) ** 2 for x in self.epoch_times) / len( + self.epoch_times + ) + variance = math.sqrt(variance) + + sorted_list = sorted(self.epoch_times) + p90_val = sorted_list[int(len(sorted_list) * 0.9)] + p99_val = sorted_list[int(len(sorted_list) * 0.99)] + header = f"{'Init time':<18} {'Max iteration time':<20} {'Min iteration time':<20} {'Avg iteration time':<20} {'P90 iteration time ':<20} {'Iteration time Std ':<20}\n" + separator = "-" * len(header) + "\n" + log_str = separator + header + separator + iteration_result = f"{self.epoch_times[0]:<18.2f} {max_val:<20.2f} {min_val:<20.2f} {mean_val:<20.2f} {p90_val:<20.2f} {variance:<20.2f}\n" + log_str += iteration_result + print_fn(f"\n\tDetailed info for AICB iteration time\n{log_str}") + + +class Workload: + def __init__(self) -> None: + self.workload = [] + + def append(self, log_item: Union[LogItem, Dict]): + if isinstance(log_item, LogItem): + self.workload.append(log_item) + return + if "stage" not in log_item: + log_item["stage"] = log_item["operation"] if "operation" in log_item else "" + if "comm_group" not in log_item: + assert ( + log_item["comm_type"] == CommType.computation + ), "comm_group is required for non-computation comm_type" + log_item["comm_group"] = CommGroup.all + self.workload.append( + LogItem( + comm_type=log_item["comm_type"], + comm_group=log_item["comm_group"], + comm_group_size=log_item["comm_group_size"], + msg_size=log_item["msg_size"], + stage=log_item["stage"], + src=log_item.get("src", None), + dst=log_item.get("dst", None), + additional=log_item.get("additional", None), + ) + ) + + def extend(self, new_workload): + self.workload.extend(new_workload.workload) + + def dump(self, filename): + folder_path = os.path.dirname(filename) + if folder_path and not os.path.exists(folder_path): + os.makedirs(folder_path) + default_folder_path = "results/mocked_workload/" + if not os.path.exists(default_folder_path): + os.makedirs(default_folder_path, exist_ok=True) + if "." in filename: + filename = os.path.basename(filename).split(".")[0] + filename = os.path.join("results/mocked_workload/", filename) + csv_filename = filename + "_workload.csv" + + rank_gen = getattr(self, 'rank_generator', None) + + if rank_gen is not None: + from utils.rank_mapper import get_rank_list_for_comm_group, build_rank_mapping_table + for log_item in self.workload: + if log_item.ranks is not None: + continue + if log_item.comm_group is None: + continue + log_item.ranks = get_rank_list_for_comm_group( + rank_gen, log_item.comm_group, log_item.comm_group_size + ) + if log_item.comm_group_size is None and log_item.ranks: + log_item.comm_group_size = len(log_item.ranks) + logger.warning( + "Rank auto-populated at dump time for LogItem stage=%s, comm_group=%s.", + log_item.stage, log_item.comm_group.value, + ) + + mapping_rows = build_rank_mapping_table(rank_gen) + sidecar_filename = filename + "_rank_mapping.csv" + with open(sidecar_filename, "w") as sf: + sf.write("group,size,rank_groups\n") + for row in mapping_rows: + rank_groups_quoted = '"' + row["rank_groups"] + '"' + sf.write(f"{row['group']},{row['size']},{rank_groups_quoted}\n") + print(f"Rank mapping file generated:{sidecar_filename}") + else: + logger.warning("No rank_generator available; skipping rank population and sidecar.") + + with open(csv_filename, "w") as f: + f.write(self.workload[0].csv_header() + "\n") + for log_item in self.workload: + log_item_write = copy.deepcopy(log_item) + if(log_item_write.comm_type == CommType.computation): + msg_size_str = "("+' '.join(str(shape).replace(',', '') for shape in log_item_write.msg_size)+")" + log_item_write.msg_size = msg_size_str + f.write(log_item_write.view_as_csv_line() + "\n") + del log_item_write + print(f"Workload file generated:{csv_filename}") + + @staticmethod + def load(filename): + filename = filename.split(".") + filename[-1] = "pkl" + filename = ".".join(filename) + workload, args = pickle.load(open(filename, "rb")) + return workload, args diff --git a/aicb/log_analyzer/plot.py b/aicb/log_analyzer/plot.py new file mode 100644 index 000000000..a80a22286 --- /dev/null +++ b/aicb/log_analyzer/plot.py @@ -0,0 +1,86 @@ +""" +Copyright (c) 2021, Alibaba Group; +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import numpy as np +import matplotlib.pyplot as plt +from log_analyzer.ds_comm_log_analyzer import parse_ds_comm_log +from utils.benchmark_logger import BenchLogger +from typing import Dict, List +from log_analyzer.utils import convert_size_to_msg + + +def log_boxplot(detailed_comm_info: Dict): + MAX_ITEMS, COLS = 5, 2 + comm_type2msg_size2time_cost = {} + for comm_type, comm_group, msg_size in sorted(detailed_comm_info.keys()): + if (comm_type, comm_group) not in comm_type2msg_size2time_cost: + comm_type2msg_size2time_cost[(comm_type, comm_group)] = {} + elasped_time = np.array( + detailed_comm_info[(comm_type, comm_group, msg_size)]["_elapsed_time"] + ) + comm_type2msg_size2time_cost[(comm_type, comm_group)][ + msg_size + ] = elasped_time # [elasped_time < 3000] + fig_num = sum( + [ + (len(comm_info.keys()) + MAX_ITEMS - 1) // MAX_ITEMS + for comm_info in comm_type2msg_size2time_cost.values() + ] + ) + + fig_rows, fig_idx = (fig_num + COLS - 1) // COLS, 0 + fig, axes = plt.subplots(nrows=fig_rows, ncols=COLS, figsize=(8, 6)) + fig.tight_layout() + fig.suptitle("for deepspeed Zero3 llama 13B") + for (comm_type, comm_group), comm_info in comm_type2msg_size2time_cost.items(): + values, labels = list(comm_info.values()), [ + convert_size_to_msg(msg) for msg in comm_info.keys() + ] + for j in range(0, len(values), MAX_ITEMS): + ax = axes[fig_idx // COLS][fig_idx % COLS] + fig_idx += 1 + ax.set_title("%s %s msg info" % (comm_type.value, comm_group.value)) + ax.boxplot( + values[j : j + MAX_ITEMS], + labels=labels[j : j + MAX_ITEMS], + flierprops=dict( + marker="o", markerfacecolor="black", markersize=2, linestyle="none" + ), + ) + for k in range(j, min(j + MAX_ITEMS, len(values))): + ax.text( + x=k - j + 1, + y=np.max(values[k]) * 1.01, + s=len(values[k]), + horizontalalignment="center", + size="x-small", + color="r", + weight="semibold", + ) + plt.show() + + +def log_time_plotter(epoch_times: List[float]): + plt.plot(epoch_times) + plt.show() + + +if __name__ == "__main__": + filename = "/Users/yikaizhu/alicode/models-perf/deepspeed_baichuan_exp/results/baichuan_13B_zero3/55n_915_comm_log.txt" + filename = "/Users/yikaizhu/Desktop/AIBC_clean.txt" + comm_log = parse_ds_comm_log(filename) + comm_info = comm_log.analyze() + if "train" in comm_info: + log_boxplot(comm_info["train"]["detailed_comm_type_info"]) + else: + log_boxplot(comm_info["init"]["detailed_comm_type_info"]) diff --git a/aicb/log_analyzer/utils.py b/aicb/log_analyzer/utils.py new file mode 100644 index 000000000..56210c1f6 --- /dev/null +++ b/aicb/log_analyzer/utils.py @@ -0,0 +1,63 @@ +""" +Copyright (c) 2021, Alibaba Group; +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import math +from utils.utils import CommGroup, CommType, get_args + + +def convert_size_to_msg(size_bytes): + if size_bytes == 0: + return "0 B" + size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") + i = int(math.floor(math.log(size_bytes, 1024))) + p = math.pow(1024, i) + s = round(size_bytes / p, 2) + return "%s %s" % (s, size_name[i]) + + +def convert_msg_to_size(msg): + if msg == "0B": + return 0 + try: + num, name = msg.split(" ") + except: + print(f"cannot convert msg into int") + return 0 + num, name = float(num), name.strip() + size_name = ["B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"] + if name not in size_name: + return None + p = math.pow(1024, size_name.index(name)) + return num * p + + +def calc_bw_log(comm_type: CommType, size, duration,group_size): # size: Bytes; duration: ms + n = group_size if group_size else 1 + duration /= 1000 + if comm_type in [CommType.all_gather, CommType.reduce_scatter]: + # size *= n + tput = size / duration + busbw = (size / duration) * ((n - 1) / n) + elif comm_type == CommType.all_reduce: + tput = size / duration + busbw = (size / duration) * (2 * (n - 1) / n) + elif comm_type in [CommType.barrier, CommType.computation]: + return 0, 0 + else: # [CommType.broadcast, CommType.reduce, "gather", "scatter", "isend", "irecv"] + tput = size / duration + busbw = tput + tput /= 1024*1024*1024 + busbw /= 1024*1024*1024 + tput = round(tput, 2) + busbw = round(busbw, 2) + return tput, busbw diff --git a/aicb/requirements.txt b/aicb/requirements.txt new file mode 100644 index 000000000..8e099321a --- /dev/null +++ b/aicb/requirements.txt @@ -0,0 +1,17 @@ +# requirements.txt + +# Core dependencies from PyPI +transformers==4.57.1 +msgspec>=0.19.0 +blake3>=1.0.4 +fastapi==0.119.0 +openai==2.3.0 +py-cpuinfo>=9.0.0 +einops +vllm==0.11.0 + +# Dependencies installed from Git repositories +git+https://github.com/fanshiqing/grouped_gemm@v1.0 +git+https://github.com/deepseek-ai/DeepGEMM.git@594953acce41793ae00a1233eb516044d604bcb6 +git+https://github.com/deepseek-ai/FlashMLA.git@1408756a88e52a25196b759eaf8db89d2b51b5a1 +git+https://github.com/flashinfer-ai/flashinfer.git@9c264c0b5ad76593d7d343f4a5711a4fa6fdc909 \ No newline at end of file diff --git a/aicb/run_suites.py b/aicb/run_suites.py new file mode 100755 index 000000000..941d5aa9b --- /dev/null +++ b/aicb/run_suites.py @@ -0,0 +1,173 @@ +""" +Copyright (c) 2021, Alibaba Group; +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import sys +import subprocess +import os +import configparser +import argparse + +running_command = {} +default_config = { + "deepspeed": { + "llama7b_zero2": 0, + "llama65b_zero3": 0, + }, + "megatron": { + "llama_7B": 0, + "gpt_13B_sp": 0, + "gpt_175B_tp":0, + "gpt_175B": 0, + "gpt_22B": 0, + "llama_405B": 0, + "Mixtral_8*7B": 0, + }, + "aiob" : { #aicb workload suites with computation + "llama_7B_aiob": 0, + "gpt_13B_sp_aiob": 0, + "gpt_175B_aiob": 0, + "gpt_22B_aiob": 0, + "gpt_175B_tp_aiob": 0, + "llama_405B_aiob": 0, + "Mixtral_8*7B_aiob": 0, + "llama7B_zero2_aiob": 0, + "llama65B_zero3_aiob": 0, + }, + "coll_comm_check": {"all_reduce": 0, "all_gather": 0, "muti_all_reduce": 0}, +} + + +def get_params(): + parser = argparse.ArgumentParser() + parser.add_argument("--output", help="output directory", default="./results") + + +def read_config(config): + ds_conf = config["deepspeed"] + megatron_conf = config["megatron"] + cc_conf = config["coll_comm_check"] + aiob_conf = config["aiob"] + if int(ds_conf["llama7b_zero2"]): + running_command["deepspeed2_llama13b"] = ( + f"bash scripts/deepspeed_llama.sh --zero_stage 2 -m 13 --epoch_num 10 " + ) + if int(ds_conf["llama65b_zero3"]): + running_command["deepspeed3_llama65b"] = ( + f"bash scripts/deepspeed_llama.sh --zero_stage 3 -m 65 --epoch_num 10 \ + --reduce_bucket_size 1000000000 --allgather_bucket_size 500000000 \ + --param_persistence_threshold 1000000" + ) + if int(megatron_conf["llama_7B"]): + running_command["megatron_llama7B"] = ( + f"bash scripts/megatron_gpt.sh -m 7 --tensor_model_parallel_size 1 --epoch_num 10 --seq_length 4096" + ) + if int(megatron_conf["gpt_13B_sp"]): + running_command["megatron_gpt13b_sp"] = ( + f"bash scripts/megatron_gpt.sh -m 13 --tensor_model_parallel_size 2 --epoch_num 10 --sp" + ) + if int(megatron_conf["gpt_175B"]): + running_command["megatron_gpt175B"] = ( + f"bash scripts/megatron_gpt.sh -m 175 --tensor_model_parallel_size 8 --epoch_num 10 --pipeline_model_parallel 2 --sp" + ) + if int(megatron_conf["gpt_175B_tp"]): + running_command["megatron_gpt175B_tp"] = ( + f"bash scripts/megatron_gpt.sh -m 175 --tensor_model_parallel_size 8 --epoch_num 10 --pipeline_model_parallel 2" + ) + if int(megatron_conf["gpt_22B"]): + running_command["megatron_gpt_22B"] = ( + f"bash scripts/megatron_gpt.sh -m 22 --tensor_model_parallel_size 4 --epoch_num 10 --sp" + ) + if int(megatron_conf["Mixtral_8*7B"]): + running_command["megatron_moe"] = ( + f"bash scripts/megatron_gpt.sh -m moe --tensor_model_parallel_size 2 --epoch_num 10 --sp --ep 4 --num_experts 16 --topk 4 " + ) + if int(megatron_conf["llama_405B"]): + running_command["megatron_llama_405B"] = ( + f"bash scripts/megatron_gpt.sh -m 405 --tensor_model_parallel_size 8 --epoch_num 10 --sp --seq_length 8192" + ) + if int(aiob_conf["llama_7B_aiob"]): + running_command["megatron_llama7b_aiob"] = ( + f"bash scripts/megatron_gpt.sh -m 7 --epoch_num 10 --aiob_enable " + ) + if int(aiob_conf["gpt_13B_sp_aiob"]): + running_command["megatron_gpt13b_sp_aiob"] = ( + f"bash scripts/megatron_gpt.sh -m 13 --tensor_model_parallel_size 4 --epoch_num 10 --aiob_enable --sp" + ) + if int(aiob_conf["gpt_175B_aiob"]): + running_command["megatron_gpt175B_aiob"] = ( + f"bash scripts/megatron_gpt.sh -m 175 --tensor_model_parallel_size 8 --epoch_num 10 --aiob_enable --pipeline_model_parallel 2 --sp" + ) + if int(aiob_conf["gpt_175B_tp_aiob"]): + running_command["megatron_gpt175B_tp_aiob"] = ( + f"bash scripts/megatron_gpt.sh -m 175 --tensor_model_parallel_size 8 --epoch_num 10 --aiob_enable --pipeline_model_parallel 2 " + ) + if int(aiob_conf["gpt_22B_aiob"]): + running_command["megatron_gpt_22B_aiob"] = ( + f"bash scripts/megatron_gpt.sh -m 22 --tensor_model_parallel_size 4 --epoch_num 10 --aiob_enable --sp" + ) + if int(aiob_conf["Mixtral_8*7B_aiob"]): + running_command["megatron_moe_aiob"] = ( + f"bash scripts/megatron_gpt.sh -m moe --tensor_model_parallel_size 2 --epoch_num 10 --sp --ep 4 --num_experts 16 --topk 4 --aiob_enable " + ) + if int(aiob_conf["llama_405B_aiob"]): + running_command["megatron_llama_405B"] = ( + f"bash scripts/megatron_gpt.sh -m 405 --tensor_model_parallel_size 8 --epoch_num 10 --sp --seq_length 8192 --aiob_enable " + ) + if int(aiob_conf["llama7B_zero2_aiob"]): + running_command["deepspeed2_llama7b_aiob"] = ( + f"bash scripts/deepspeed_llama.sh --zero_stage 2 -m 7 --epoch_num 10 --aiob_enable " + ) + if int(aiob_conf["llama65B_zero3_aiob"]): + running_command["deepspeed3_llama65b_aiob"] = ( + f"bash scripts/deepspeed_llama.sh --zero_stage 3 -m 65 --epoch_num 10 \ + --reduce_bucket_size 1000000000 --allgather_bucket_size 500000000 \ + --param_persistence_threshold 1000000 --aiob_enable" + ) + if int(cc_conf["all_reduce"]): + running_command["all_reduce_check"] = ( + f"bash scripts/coll_comm_check.sh --iter_num 100 --test_comm all_reduce --model_name all_reduce" + ) + if int(cc_conf["all_gather"]): + running_command["all_gather_check"] = ( + f"bash scripts/coll_comm_check.sh --iter_num 100 --test_comm all_gather --model_name all_gather" + ) + if int(cc_conf["muti_all_reduce"]): + running_command["muti_all_reduce_check"] = ( + f"bash scripts/coll_comm_check.sh --iter_num 100 --test_comm all_reduce --model_name muti_all_reduce --muti_all_reduce_enable 1" + ) + + + +if __name__ == "__main__": + # config = configparser.ConfigParser() + # config.read('config.ini') + read_config(config=default_config) + result = {} + print(running_command) + for name, command in running_command.items(): + result_dir = "./results/log/" + if not os.path.isdir(result_dir): + os.makedirs(result_dir) + output_file = f"./results/log/{name}.txt" + + command += f" 2>&1 | tee {output_file}" + print(name) + ret = subprocess.run(command, shell=True, text=True) + + # if ret.returncode != 0: + # print(f"ERROR when running {name}: {command}") + # print( + # f"return state is {ret.returncode}, got err{ret.stderr}, get output{ret.stdout}" + # ) + # exit(-1) + # command_out = ret.stdout + # print(command_out) diff --git a/aicb/scripts/coll_comm_check.sh b/aicb/scripts/coll_comm_check.sh new file mode 100644 index 000000000..ea624f656 --- /dev/null +++ b/aicb/scripts/coll_comm_check.sh @@ -0,0 +1,94 @@ +#!/bin/sh + +set -x + +begin_size=4096 +end_size=8589934592 +epoch_num=1 +iter_num=500 +test_comm=all_reduce +frame=collective_test +model_name=all_reduce +multi_all_reduce_enable=0 + +usage() { + echo "Usage: $0 [options] + options: + --iter_num num of iterations: $iter_num + --begin_size start message size of test: $begin_size + --end_size end message size of test: $end_size + --test_comm collective communication type: $test_comm + --multi_all_reduce_enable enable muti all_reduce opration: $multi_all_reduce_enable + -h, --help" 1>&2; exit 1; +} + +while [ $# -gt 0 ] +do + case $1 in + --model_name|--model-name) + model_name=$2; shift;; + --iter-num|--iter_num) + iter_num=$2 ; shift;; + --begin_size|--begin-size) + begin_size=$2 ; shift;; + --end_size|--end-size) + end_size=$2 ; shift;; + --test_comm|--test-comm) + test_comm=$2 ; shift;; + --multi_all_reduce_enable|--muti-all-reduce-enable) + multi_all_reduce_enable=$2 ; shift;; + -h|--help) + usage ;; + (*) + break;; + esac + # Fetch next argument as 1st + shift +done + + +script="./aicb.py" + +if [ "$multi_all_reduce_enable" -eq 0 ]; then + echo "torchrun \ + --nnodes $WORLD_SIZE \ + --node_rank $RANK \ + --nproc_per_node gpu \ + --master_addr $MASTER_ADDR \ + --master_port $MASTER_PORT \ + $script --iter_num=$iter_num --world_size=$((WORLD_SIZE*8))\ + --begin_size=$begin_size --end_size=$end_size --test_comm=$test_comm --model_name=$model_name\ + --frame=standard_check --multi_all_reduce_enable=$multi_all_reduce_enable" +else + echo "torchrun \ + --nnodes $WORLD_SIZE \ + --node_rank $RANK \ + --nproc_per_node gpu \ + --master_addr $MASTER_ADDR \ + --master_port $MASTER_PORT \ + $script --iter_num=$iter_num --world_size=$((WORLD_SIZE*8))\ + --begin_size=$begin_size --end_size=$end_size --test_comm=$test_comm --model_name=$model_name\ + --frame=standard_check --multi_all_reduce_enable=$multi_all_reduce_enable --pipeline_model_parallel=$WORLD_SIZE" +fi + +if [ "$multi_all_reduce_enable" -eq 0 ]; then + torchrun \ + --nnodes $WORLD_SIZE \ + --node_rank $RANK \ + --nproc_per_node gpu \ + --master_addr $MASTER_ADDR \ + --master_port $MASTER_PORT \ + $script --iter_num=$iter_num --world_size=$((WORLD_SIZE*8))\ + --begin_size=$begin_size --end_size=$end_size --test_comm=$test_comm --model_name=$model_name\ + --frame=collective_test --multi_all_reduce_enable=$multi_all_reduce_enable +else + torchrun \ + --nnodes $WORLD_SIZE \ + --node_rank $RANK \ + --nproc_per_node gpu \ + --master_addr $MASTER_ADDR \ + --master_port $MASTER_PORT \ + $script --iter_num=$iter_num --world_size=$((WORLD_SIZE*8))\ + --begin_size=$begin_size --end_size=$end_size --test_comm=$test_comm --model_name=$model_name\ + --frame=collective_test --multi_all_reduce_enable=$multi_all_reduce_enable --pipeline_model_parallel=$WORLD_SIZE +fi \ No newline at end of file diff --git a/aicb/scripts/deepspeed_llama.sh b/aicb/scripts/deepspeed_llama.sh new file mode 100644 index 000000000..f51c6a3dc --- /dev/null +++ b/aicb/scripts/deepspeed_llama.sh @@ -0,0 +1,115 @@ +#!/bin/sh + + +set -x +: ${WORLD_SIZE:=1} +: ${RANK:=0} +: ${MASTER_ADDR:="localhost"} +: ${MASTER_PORT:=29500} +model_name=llama_7b +zero_stage=3 +model_size=7 +num_layers=32 +epoch_num=10 +num_attention_heads=32 +hidden_size=4096 +ffn_hidden_size=11008 +reduce_bucket_size=500000000 +allgather_bucket_size=500000000 +prefetch_bucket_size=1000000000 +max_live_parameters=1000000000 +param_persistence_threshold=100000 +seq_len=2048 +batch_size=4 +contiguous_gradients= +aiob_enable= +enable_visual= +workload_only= + +usage() { + echo "Usage: $0 [options] + options: + --model_name model_name: $model_name + --zero_stage zero_stage: $zero_stage + --epoch_num num of iterations: $epoch_num + --batch_size micro batch_size: $batch_size + --enable_visual enable visual html output files + --workload_only generate workload only + -m, --model-size llama model size.(7/13/30/65): $model_size + --reduce-bucket-size size of reduce bucket: $reduce_bucket_size + --allgather-bucket-size size of all_gather bucket(only used in stage1,2): $reduce_bucket_size + --prefetch-bucket-size size of all_gather prefetch bucket(only used in stage3): $prefetch_bucket_size + --max-live-parameters max size of params that have been all_gather(only used in stage3): $max_live_parameters + --param-persistence-threshold threshold of param that is all-gather before forward(only used in stage3): $param_persistence_threshold + --seq-len seq-len: $seq_len + --contiguous-gradients use reduce instead of all_reduce (only used in stage2) + -h, --help" 1>&2; exit 1; +} + +while [ $# -gt 0 ] +do +echo "Processing argument: $1" + case $1 in + --model_name|--model-name) + model_name=$2 ; shift;; + --stage|--zero-stage|--zero_stage) + zero_stage=$2 ; shift;; + --epoch-num|--epoch_num) + epoch_num=$2 ; shift;; + --batch-size|--micro_batch|--batch_size) + batch_size=$2 ; shift;; + -m|--model-size) + model_size=$2 ; shift;; + --reduce-bucket-size|--reduce_bucket_size) + reduce_bucket_size=$2 ; shift;; + --param-persistence-threshold|--param_persistence_threshold) + param_persistence_threshold=$2 ; shift;; + --max-live-parameters|--max_live_parameters) + prefetch_bucket_size=$2 ; shift;; + --allgather-bucket-size|--allgather_bucket_size) + allgather_bucket_size=$2 ; shift;; + --seq-len|--seq_len) + seq_len=$2 ; shift;; + --aiob_enable) + aiob_enable=--aiob_enable;; + --enable_visual) + enable_visual=--enable_visual;; + --workload_only) + workload_only=--workload_only;; + --contiguous-gradients|--contiguous_gradients) + contiguous_gradients=--contiguous_gradients; shift;; + -h|--help) + usage ;; + (*) + break;; + esac + # Fetch next argument as 1st + shift +done + +case $model_size in + 13) + model_name=llama_13b hidden_size=5120; ffn_hidden_size=13824; num_layers=40; num_attention_heads=40; shift;; + 30) + model_name=llama_30b hidden_size=6656; ffn_hidden_size=17920; num_layers=60; num_attention_heads=52; shift;; + 65) + model_name=llama_65b hidden_size=8192; ffn_hidden_size=22016; num_layers=80; num_attention_heads=64; shift;; + 7) + ;; + (*) + echo "only suport model size 7b, 13b, 30b, 65b, got $model_size";; +esac + +script="./aicb.py" + +torchrun \ +--nnodes ${WORLD_SIZE} \ +--node_rank $RANK \ +--nproc_per_node gpu \ +--master_addr $MASTER_ADDR \ +--master_port $MASTER_PORT \ +$script --frame=DeepSpeed --model_name=$model_name --stage=$zero_stage --world_size=$((WORLD_SIZE*8)) \ + --micro_batch=$batch_size --global_batch=$((WORLD_SIZE*8*batch_size)) --epoch_num=$epoch_num \ + --num_layers=$num_layers --hidden_size=$hidden_size --ffn_hidden_size=$ffn_hidden_size --num_attention_heads=$num_attention_heads \ + --reduce_bucket_size=$reduce_bucket_size --allgather_bucket_size=$allgather_bucket_size --seq_len=$seq_len \ + --max_live_parameters=$max_live_parameters --param_persistence_threshold=$param_persistence_threshold $contiguous_gradients $aiob_enable $enable_visual $workload_only \ No newline at end of file diff --git a/aicb/scripts/generate_per_rank_csv.py b/aicb/scripts/generate_per_rank_csv.py new file mode 100644 index 000000000..2c316904b --- /dev/null +++ b/aicb/scripts/generate_per_rank_csv.py @@ -0,0 +1,113 @@ +"""Generate per-rank CSV files from a rank-0 workload CSV. + +Auto-detects tp/pp/ep/world_size from the sidecar _rank_mapping.csv. +Can also override via CLI arguments. + +Usage: + cd aicb && python3 scripts/generate_per_rank_csv.py --csv results/.../xxx_workload.csv +""" + +import argparse, csv, os, sys +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from utils.utils import RankGenerator, CommGroup +from utils.rank_mapper import get_rank_list_for_comm_group + + +def _detect_from_sidecar(sidecar_path): + if not os.path.exists(sidecar_path): + return {} + info = {} + with open(sidecar_path, newline='') as f: + for row in csv.reader(f): + if not row or row[0] == 'group': + continue + name, size_str = row[0], row[1] + size = int(size_str) + if name == 'all_nodes': info['world_size'] = size + elif name == 'tp_group': info['tp'] = size + elif name == 'pp_group': info['pp'] = size + elif name == 'dp_group': info['dp_full'] = size + elif name == 'ep_group': + rank_groups = row[2].split() + info['ep'] = size if len(rank_groups) > 0 and size > 1 else 1 + return info + + +def main(): + parser = argparse.ArgumentParser(description="Generate per-rank workload CSVs") + parser.add_argument("--csv", required=True) + parser.add_argument("--tp", type=int, default=None) + parser.add_argument("--pp", type=int, default=None) + parser.add_argument("--ep", type=int, default=None) + parser.add_argument("--cp", type=int, default=1) + parser.add_argument("--order", default="tp-cp-ep-dp-pp") + parser.add_argument("--outdir", default=None) + args = parser.parse_args() + + sidecar = args.csv.replace('_workload.csv', '_rank_mapping.csv') + info = _detect_from_sidecar(sidecar) + world_size = info.get('world_size') + tp = args.tp or info.get('tp', 1) + pp = args.pp or info.get('pp', 1) + ep = args.ep or info.get('ep', 1) + + if world_size is None: + with open(args.csv, newline='') as f: + reader = csv.reader(f) + header = next(reader) + ranks_col = len(header) - 1 + max_rank = 0 + for row in reader: + r = row[ranks_col] if len(row) > ranks_col else '' + if r: + for x in r.strip('"').split(','): + try: max_rank = max(max_rank, int(x)) + except: pass + world_size = max(max_rank + 1, tp * pp) + + dp = world_size // (tp * pp) + rg = RankGenerator(tp=tp, ep=ep, dp=dp, pp=pp, cp=args.cp, order=args.order) + + print(f"World: {rg.world_size} | tp={tp} dp={dp} pp={pp} ep={ep}") + if info: print(f"Auto-detected from: {sidecar}") + + with open(args.csv, newline='') as f: + reader = csv.reader(f) + all_rows = list(reader) + header = all_rows[0] + data_rows = all_rows[1:] + ranks_col = len(header) - 1 + print(f"Input: {args.csv} ({len(data_rows)} rows)") + + outdir = args.outdir or os.path.dirname(args.csv) or '.' + os.makedirs(outdir, exist_ok=True) + base_name = os.path.basename(args.csv).replace('_workload.csv', '') + + for ref_rank in range(rg.world_size): + out_file = os.path.join(outdir, f"{base_name}_rank{ref_rank}_workload.csv") + with open(out_file, 'w', newline='') as f: + writer = csv.writer(f) + writer.writerow(header) + for row in data_rows: + cg_raw = row[1] if len(row) > 1 else '' + if cg_raw in ('None', '', 'CommGroup.None'): + writer.writerow(row) + continue + cg_name = cg_raw.split('.')[-1] + try: cg = CommGroup(cg_name) + except ValueError: writer.writerow(row); continue + new_ranks = get_rank_list_for_comm_group(rg, cg, ref_rank=ref_rank) + new_row = list(row) + while len(new_row) <= ranks_col: + new_row.append('') + new_row[ranks_col] = '"' + ','.join(str(r) for r in new_ranks) + '"' + writer.writerow(new_row) + tag = f" [{ref_rank}] {out_file}" + if ref_rank < 2 or ref_rank >= rg.world_size - 1: print(tag) + elif ref_rank == 2: print(" ...") + + print(f"\nDone. {rg.world_size} rank CSVs written to {outdir}/") + + +if __name__ == "__main__": + main() diff --git a/aicb/scripts/inference_configs/.gitignore b/aicb/scripts/inference_configs/.gitignore new file mode 100644 index 000000000..e4d340ceb --- /dev/null +++ b/aicb/scripts/inference_configs/.gitignore @@ -0,0 +1 @@ +generated_configs/ \ No newline at end of file diff --git a/aicb/scripts/inference_configs/deepseek_default.json b/aicb/scripts/inference_configs/deepseek_default.json new file mode 100644 index 000000000..2e1e96184 --- /dev/null +++ b/aicb/scripts/inference_configs/deepseek_default.json @@ -0,0 +1,21 @@ +{ + "num_layers": 61, + "dense_layer": 3, + "hidden_size": 7168, + "num_attention_heads": 128, + "vocab_size": 129280, + "d_kv_c": 512, + "d_q_c": 1536, + "d_r": 64, + "head_num": 128, + "d_q": 128, + "d_kv": 128, + "router_expert": 256, + "duped_expert": 32, + "shared_experts": 1, + "moe_router_topk": 8, + "expert_dim": 2048, + "num_experts": 288, + "computation_enable": true, + "add_bias_linear": false +} \ No newline at end of file diff --git a/aicb/scripts/inference_configs/qwen3_moe_default.json b/aicb/scripts/inference_configs/qwen3_moe_default.json new file mode 100644 index 000000000..5366b39a2 --- /dev/null +++ b/aicb/scripts/inference_configs/qwen3_moe_default.json @@ -0,0 +1,38 @@ +{ + "architectures": [ + "Qwen3MoeForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "decoder_sparse_step": 1, + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "max_position_embeddings": 40960, + "max_window_layers": 94, + "mlp_only_layers": [], + "model_type": "qwen3_moe", + "moe_intermediate_size": 1536, + "norm_topk_prob": true, + "num_attention_heads": 64, + "num_experts": 128, + "num_experts_per_tok": 8, + "num_hidden_layers": 94, + "num_key_value_heads": 4, + "output_router_logits": false, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000.0, + "router_aux_loss_coef": 0.001, + "sliding_window": null, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.51.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} \ No newline at end of file diff --git a/aicb/scripts/inference_configs/qwen3_next_default.json b/aicb/scripts/inference_configs/qwen3_next_default.json new file mode 100644 index 000000000..e8dad757f --- /dev/null +++ b/aicb/scripts/inference_configs/qwen3_next_default.json @@ -0,0 +1,43 @@ +{ + "architectures": [ + "Qwen3NextForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "decoder_sparse_step": 1, + "eos_token_id": 151645, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 5120, + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_value_head_dim": 128, + "max_position_embeddings": 262144, + "mlp_only_layers": [], + "model_type": "qwen3_next", + "moe_intermediate_size": 512, + "norm_topk_prob": true, + "num_attention_heads": 16, + "num_experts": 512, + "num_experts_per_tok": 10, + "num_hidden_layers": 48, + "num_key_value_heads": 2, + "output_router_logits": false, + "partial_rotary_factor": 0.25, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 10000000, + "router_aux_loss_coef": 0.001, + "shared_expert_intermediate_size": 512, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.57.0.dev0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} \ No newline at end of file diff --git a/aicb/scripts/inference_workload_with_aiob.sh b/aicb/scripts/inference_workload_with_aiob.sh new file mode 100644 index 000000000..3787c1b75 --- /dev/null +++ b/aicb/scripts/inference_workload_with_aiob.sh @@ -0,0 +1,197 @@ +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) + +model_size=deepseek-671B +config_file_path= +phase=decode +seq_length=1024 +micro_batch=32 +world_size=8 +tensor_model_parallel_size=1 +expert_model_parallel_size=8 +pipeline_model_parallel=1 +moe_enable=true +result_dir=results/workload/ +aiob_enable=false +aiob_forward_loops=10 + +dpsk_default_path="$SCRIPT_DIR/inference_configs/deepseek_default.json" +qwen3_moe_default_path="$SCRIPT_DIR/inference_configs/qwen3_moe_default.json" +qwen3_next_default_path="$SCRIPT_DIR/inference_configs/qwen3_next_default.json" + +usage() { + cat <<-EOF +Usage: $0 [OPTIONS] + + Generate inference workload with AIOB support. + +Options: + -m, --model-size + Model size to use. + Possible values: {deepseek-671B, qwen3-235B, qwen3-next-80B}. + (Default: $model_size) + + -c, --config + Path to a custom configuration file. + (Default: None) + + -p, --phase + Inference phase. + Possible values: {prefill, decode}. + (Default: $phase) + + -s, --seq-length + Sequence length for the model. + (Default: $seq_length) + + -b, --micro-batch + Micro batch size. + (Default: $micro_batch) + + -w, --world-size + Total number of GPUs (world size). + (Default: $world_size) + + -t, --tp-size + Tensor model parallel size. + (Default: $tensor_model_parallel_size) + + -e, --ep-size + Expert model parallel size (for MoE models). + (Default: $expert_model_parallel_size) + + -l, --pp-size + Pipeline model parallel size. + (Default: $pipeline_model_parallel) + + -M, --moe-enable + Enable MoE (Mixture of Experts) support. + (This is a boolean flag. Default: $moe_enable) + + -r, --result-dir + Directory to save the results. + (Default: "$result_dir") + + -a, --aiob-enable + Enable AIOB (All-In-One Block) support. + (This is a boolean flag. Default: $aiob_enable) + + -f, --aiob-loops + Number of forward loops for AIOB. + (Default: $aiob_forward_loops) + + -h, --help + Display this help message and exit. + +Example: + sh $0 -m deepseek-671B -p decode -s 1024 -b 32 --aiob-enable + +EOF + exit 1 +} + +while [ $# -gt 0 ] +do + case $1 in + -m|--model_size) + model_size=$2; shift;; + -c|--config) + config_file_path=$2; shift;; + -p|--phase) + phase=$2; shift;; + -s|--seq_length) + seq_length=$2; shift;; + -b|--micro_batch) + micro_batch=$2; shift;; + -w|--world_size) + world_size=$2; shift;; + -t|--tensor_model_parallel_size) + tensor_model_parallel_size=$2; shift;; + -e|--expert_model_parallel_size) + expert_model_parallel_size=$2; shift;; + -l|--pipeline_model_parallel) + pipeline_model_parallel=$2; shift;; + -M|--moe_enable) + moe_enable=true;; + -r|--result_dir) + result_dir=$2; shift;; + -a|--aiob_enable) + aiob_enable=true;; + -f|--aiob_forward_loops) + aiob_forward_loops=$2; shift;; + -h|--help) + usage;; + (*) + break;; + esac + shift +done + +case $model_size in + deepseek-671B) + model_name=DeepSeek-671B + config_file_path=${config_file_path:-$dpsk_default_path} + ;; + qwen3-235B) + model_name=Qwen3-Moe-235B + config_file_path=${config_file_path:-$qwen3_moe_default_path} + ;; + qwen3-next-80B) + model_name=Qwen3-Next-80B + config_file_path=${config_file_path:-$qwen3_next_default_path} + ;; + (*) + echo "Invalid model size: $model_size" + usage;; +esac + +# Build command with optional parameters +cmd="python -m workload_generator.SimAI_inference_workload_generator $model_name $config_file_path" + +# Add optional parameters if they are set +if [ ! -z "$phase" ]; then + cmd="$cmd --phase $phase" +fi + +if [ ! -z "$seq_length" ]; then + cmd="$cmd --seq_length $seq_length" +fi + +if [ ! -z "$micro_batch" ]; then + cmd="$cmd --micro_batch $micro_batch" +fi + +if [ ! -z "$world_size" ]; then + cmd="$cmd --world_size $world_size" +fi + +if [ ! -z "$tensor_model_parallel_size" ]; then + cmd="$cmd --tensor_model_parallel_size $tensor_model_parallel_size" +fi + +if [ ! -z "$expert_model_parallel_size" ]; then + cmd="$cmd --expert_model_parallel_size $expert_model_parallel_size" +fi + +if [ ! -z "$pipeline_model_parallel" ]; then + cmd="$cmd --pipeline_model_parallel $pipeline_model_parallel" +fi + +if [ "$moe_enable" = true ]; then + cmd="$cmd --moe_enable" +fi + +if [ ! -z "$result_dir" ]; then + cmd="$cmd --result_dir $result_dir" +fi + +if [ "$aiob_enable" = true ]; then + cmd="$cmd --aiob_enable" +fi + +if [ ! -z "$aiob_forward_loops" ]; then + cmd="$cmd --aiob_forward_loops $aiob_forward_loops" +fi + +echo $cmd + +$cmd diff --git a/aicb/scripts/megatron_gpt.sh b/aicb/scripts/megatron_gpt.sh new file mode 100755 index 000000000..8385c817e --- /dev/null +++ b/aicb/scripts/megatron_gpt.sh @@ -0,0 +1,320 @@ +#!/bin/sh + +set -x +: ${WORLD_SIZE:=1} +: ${RANK:=0} +: ${MASTER_ADDR:="localhost"} +: ${MASTER_PORT:=29500} +NUM_GPUS=$(nvidia-smi -L | wc -l) # Get the number of GPUs on a single node +model_size=13 +num_layers=40 +num_attention_heads=40 +hidden_size=5120 +seq_length=2048 +micro_batch=1 +epoch_num=1 +tensor_model_parallel_size=8 +pipeline_model_parallel=1 +vocab_size=50257 +model_name=gpt_13b +ga_num=2 +sp_enable= +frame=Megatron +aiob_enable= +max_position_embeddings=4096 +num_experts=1 +moe_enable= +enable_visual= +workload_only= +usage() { + echo "Usage: \$0 [options] + options: + --frame Communication framework: $frame + --world_size World size (number of nodes): $WORLD_SIZE + --tensor_model_parallel_size Tensor parallelism size: $tensor_model_parallel_size + --pipeline_model_parallel Pipeline parallelism size: $pipeline_model_parallel + --global_batch Global batch size: $global_batch + --micro_batch Micro batch size: $micro_batch + --num_layers Number of layers: $num_layers + --seq_length Sequence length: $seq_length + --hidden_size Hidden size: $hidden_size + --epoch_num Number of epochs: $epoch_num + --num_attention_heads Number of attention heads: $num_attention_heads + --aiob_enable Enable AIOB: $aiob_enable + --enable_visual Enable Visualization $enable_visual + --workload_only generate workload only + --use_flash_attn Use flash attention: $use_flash_attn + --swiglu Use SWIGLU: $swiglu + --ffn_hidden_size FFN hidden size: $ffn_hidden_size + --comp_filepath Computation file path: $comp_filepath + --model_name Model name: $model_name + -m, --model_size model size, defaults to $model_size (possible values: 175, 22, 13, 7) + --max_position_embeddings Max position embeddings: $max_position_embeddings + --nnodes Number of nodes: $WORLD_SIZE + --node_rank Rank of the node: $RANK + --nproc_per_node Number of GPUs per node: $NUM_GPUS + --master_addr Master address: $MASTER_ADDR + --master_port Master port: $MASTER_PORT + --me_enable enable moe + --moe_router_topk Number of experts to route to for each token. + --expert_model_parallel_size Degree of expert model parallelism + --num_experts Number of experts in the MoE model. + --moe_grouped_gemm apply grouped gemm + -h, --help Display this help and exit"1>&2; exit 1; +} +while [ $# -gt 0 ] +do +echo "Processing argument: $1" + case $1 in + --frame) + frame=$2; shift;; + --world_size) + world_size=$2; shift;; + --tensor_model_parallel_size|tp_num) + tensor_model_parallel_size=$2; shift;; + --pipeline_model_parallel|pp_num) + pipeline_model_parallel=$2; shift;; + --global_batch) + global_batch=$2; shift;; + --micro_batch) + micro_batch=$2; shift;; + --num_layers) + num_layers=$2; shift;; + --seq_length) + seq_length=$2; shift;; + --hidden_size) + hidden_size=$2; shift;; + --epoch_num) + epoch_num=$2; shift;; + --num_attention_heads) + num_attention_heads=$2; shift;; + --aiob_enable) + aiob_enable=--aiob_enable;; + --enable_visual) + enable_visual=--enable_visual;; + --workload_only) + workload_only=--workload_only;; + --use_flash_attn) + use_flash_attn=--use_flash_attn;; + --swiglu) + swiglu=--swiglu;; + --ffn_hidden_size) + ffn_hidden_size=$2; shift;; + --sp|--sp-enable|--enable_sequence_parallel) + sp_enable=--enable_sequence_parallel;; + --comp_filepath) + comp_filepath=$2; shift;; + -m|--model_size) + model_size=$2; shift;; + --moe_enable) + moe_enable=--moe_enable;; + --moe_router_topk|--topk) + moe_router_topk=$2; shift;; + --num_experts|--experts) + num_experts=$2; shift;; + --expert_model_parallel_size|--ep) + expert_model_parallel_size=$2; shift;; + --grouped_gemm|--moe_grouped_gemm) + grouped_gemm=--moe_grouped_gemm;; + --nnodes) + WORLD_SIZE=$2;shift;; + --node_rank) + RANK=$2;shift;; + --nproc_per_node) + NUM_GPUS=$2;shift;; + --master_addr) + MASTER_ADDR=$2;shift;; + --master_port) + MASTER_PORT=$2;shift;; + -h|--help) + usage ;; + (*) + break;; + esac + + shift +done + +case $model_size in + 175) + model_name=gpt_175B + num_layers=96 + hidden_size=12288 + num_attention_heads=96 + tensor_model_parallel_size=8 + ;; + 22) + model_name=gpt_22B + num_layers=48 + hidden_size=6144 + num_attention_heads=64 + tensor_model_parallel_size=8 + ;; + 13) + model_name=gpt_13B + num_layers=40 + hidden_size=5120 + num_attention_heads=40 + ;; + 7) + model_name=gpt_7B + num_layers=36 + hidden_size=4096 + num_attention_heads=32 + ;; + 405) + model_name=llama_405B + num_layers=128 + hidden_size=16384 + ffn_hidden_size=53248 + num_attention_heads=128 + tensor_model_parallel_size=8 + pipeline_model_parallel=16 + ;; + 65) + model_name=llama_65B + num_layers=80 + hidden_size=8192 + ffn_hidden_size=28672 + num_attention_heads=64 + tensor_model_parallel_size=8 + pipeline_model_parallel=2 + ;; + moe) + model_name=Mixtral_8*7B + num_layers=32 + hidden_size=4096 + num_attention_heads=32 + ffn_hidden_size=14336 + tensor_model_parallel_size=2 + moe_enable=--moe_enable + grouped_gemm=--moe_grouped_gemm + ;; + deepseek671) + model_name=DeepSeek_671B + frame=DeepSeek + moe_enable=--moe_enable + num_layers=61 + hidden_size=18432 + num_attention_heads=128 + ffn_hidden_size=2048 + q_lora_rank=1536 + kv_lora_rank=512 + qk_nope_dim=128 + qk_rope_dim=64 + v_head_dim=128 + num_experts=256 + n_shared_expert=1 + n_dense_layer=3 + tensor_model_parallel_size=1 + ;; + deepseek236) + model_name=DeepSeek_236B + frame=DeepSeek + moe_enable=--moe_enable + num_layers=60 + hidden_size=12288 + num_attention_heads=128 + ffn_hidden_size=1536 + q_lora_rank=1536 + kv_lora_rank=512 + qk_nope_dim=128 + qk_rope_dim=64 + v_head_dim=128 + num_experts=160 + n_shared_expert=2 + n_dense_layer=1 + tensor_model_parallel_size=1 + ;; + deepseek16) + model_name=DeepSeek_16B + frame=DeepSeek + moe_enable=--moe_enable + num_layers=27 + hidden_size=10944 + num_attention_heads=16 + ffn_hidden_size=1408 + q_lora_rank=0 + kv_lora_rank=512 + qk_nope_dim=128 + qk_rope_dim=64 + v_head_dim=128 + num_experts=64 + n_shared_expert=2 + n_dense_layer=1 + tensor_model_parallel_size=1 + ;; + deepseek) + frame=DeepSeek + model_name="DeepSeek" + moe_enable=--moe_enable + ;; + # set everything else from cmdline + # or it defaults to utils/utils.py's arg parser + (*) + echo "Only support model size 405,175,22,13,7 or moe; using default size 13" + model_name=gpt_13B + num_layers=40 + hidden_size=5120 + num_attention_heads=40 + ;; +esac + +dp_num=$((world_size/tensor_model_parallel_size/pipeline_model_parallel)) +global_batch=$((ga_num*dp_num*micro_batch)) +if [ $workload_only ]; then + script="python -m workload_generator.generate_megatron_workload" +else + script="./aicb.py" +fi + +cmd="$script \ + --frame=$frame \ + --model_name=$model_name \ + --world_size=$(($WORLD_SIZE * $NUM_GPUS)) \ + --tensor_model_parallel_size=$tensor_model_parallel_size \ + --micro_batch=$micro_batch \ + --global_batch=$global_batch \ + --epoch_num=$epoch_num \ + --num_layers=$num_layers \ + --hidden_size=$hidden_size \ + --num_attention_heads=$num_attention_heads \ + --seq_length=$seq_length \ + --vocab_size=$vocab_size \ + --pipeline_model_parallel=$pipeline_model_parallel \ + --use-distributed-optimizer \ + --max_position_embeddings=$max_position_embeddings \ + ${aiob_enable} \ + ${enable_visual} \ + ${workload_only} \ + ${sp_enable} \ + ${use_flash_attn} \ + ${swiglu} \ + ${ffn_hidden_size:+--ffn_hidden_size=$ffn_hidden_size} \ + ${comp_filepath:+--comp_filepath=$comp_filepath} \ + ${moe_enable} \ + ${moe_router_topk:+--moe_router_topk=$moe_router_topk} \ + ${num_experts:+--num_experts=$num_experts} \ + ${expert_model_parallel_size:+--expert_model_parallel_size=$expert_model_parallel_size} \ + ${grouped_gemm} \ + ${q_lora_rank:+--q_lora_rank=$q_lora_rank} \ + ${kv_lora_rank:+--kv_lora_rank=$kv_lora_rank} \ + ${qk_nope_dim:+--qk_nope_dim=$qk_nope_dim} \ + ${qk_rope_dim:+--qk_rope_dim=$qk_rope_dim} \ + ${v_head_dim:+--v_head_dim=$v_head_dim} \ + ${n_shared_expert:+--n_shared_expert=$n_shared_expert} \ + ${n_dense_layer:+--n_dense_layer=$n_dense_layer} \ + " +echo $cmd + +if [ $workload_only ]; then + $cmd +else + torchrun \ + --nnodes $WORLD_SIZE \ + --node_rank $RANK \ + --nproc_per_node $NUM_GPUS \ + --master_addr $MASTER_ADDR \ + --master_port $MASTER_PORT \ + $cmd +fi diff --git a/aicb/scripts/megatron_workload_with_aiob.sh b/aicb/scripts/megatron_workload_with_aiob.sh new file mode 100755 index 000000000..5404ff854 --- /dev/null +++ b/aicb/scripts/megatron_workload_with_aiob.sh @@ -0,0 +1,281 @@ +#!/bin/sh + + +frame=Megatron +world_size=32 +tensor_model_parallel_size=8 +pipeline_model_parallel=1 +global_batch=1024 +micro_batch=1 +num_layers=40 +seq_length=4096 +hidden_size=5120 +epoch_num=1 +num_attention_heads=40 +aiob_enable= +use_flash_attn= +swiglu= +sp_enable= +ffn_hidden_size= +comp_filepath= +model_size=13 +max_position_embeddings=4096 +vocab_size=50257 +num_experts=1 +moe_enable= +recompute_activations= +gpu_type=None +usage() { + echo "Usage: \$0 [options] + options: + --frame communication framework, defaults to $frame + --world_size world size, defaults to $world_size + --tensor_model_parallel_size tensor parallelism size, defaults to $tensor_model_parallel_size + --pipeline_model_parallel pipeline parallelism size, defaults to $pipeline_model_parallel + --global_batch global batch size, defaults to $global_batch + --micro_batch micro batch size, defaults to $micro_batch + --num_layers number of layers, defaults to $num_layers + --seq_length sequence length, defaults to $seq_length + --hidden_size hidden size, defaults to $hidden_size + --epoch_num number of epochs, defaults to $epoch_num + --use_distributed_optimizer use distributed optimizer + --num_attention_heads number of attention heads, defaults to $num_attention_heads + --aiob_enable enable AIOB + --use_flash_attn use flash attention + --swiglu use swiglu + --ffn_hidden_size FFN hidden size + --comp_filepath computation file path + --max_position_embeddings max position embeddings, defaults to $max_position_embeddings + -m, --model_size model size, defaults to $model_size (possible values: 175, 22, 13, 7, moe, deepseek671, deepseek236, deepseek16) + --moe_enable enable moe + --moe_router_topk Number of experts to route to for each token. + --expert_model_parallel_size Degree of expert model parallelism + --num_experts Number of experts in the MoE model. + --moe_grouped_gemm apply grouped gemm + -h, --help display this help and exit" 1>&2; exit 1; +} + + +while [ $# -gt 0 ] +do + + case $1 in + --gpu_type) + gpu_type=$2; shift;; + --frame) + frame=$2; shift;; + --world_size) + world_size=$2; shift;; + --tensor_model_parallel_size|--tp) + tensor_model_parallel_size=$2; shift;; + --pipeline_model_parallel|--pp) + pipeline_model_parallel=$2; shift;; + --global_batch) + global_batch=$2; shift;; + --micro_batch) + micro_batch=$2; shift;; + --num_layers) + num_layers=$2; shift;; + --seq_length) + seq_length=$2; shift;; + --hidden_size) + hidden_size=$2; shift;; + --epoch_num) + epoch_num=$2; shift;; + --num_attention_heads) + num_attention_heads=$2; shift;; + --aiob_enable|--aiob) + aiob_enable=--aiob_enable;; + --use_flash_attn|--flash_attn) + use_flash_attn=--use_flash_attn;; + --swiglu) + swiglu=--swiglu;; + --ffn_hidden_size) + ffn_hidden_size=$2; shift;; + --sp|--sp-enable) + sp_enable=--enable_sequence_parallel;; + --comp_filepath) + comp_filepath=$2; shift;; + -m|--model_size) + model_size=$2; shift;; + --max_position_embeddings) + max_position_embeddings=$2; shift;; + --moe_enable) + moe_enable=--moe_enable;; + --moe_router_topk|--topk) + moe_router_topk=$2; shift;; + --num_experts|--experts) + num_experts=$2; shift;; + --expert_model_parallel_size|--ep) + expert_model_parallel_size=$2; shift;; + --grouped_gemm|--moe_grouped_gemm) + grouped_gemm=--moe_grouped_gemm;; + --recompute_activations|--recompute) + recompute_activations=--recompute_activations;; + -h|--help) + usage;; + (*) + break;; + esac + shift +done + + +case $model_size in + 175) + model_name=gpt_175B + num_layers=96 + hidden_size=12288 + num_attention_heads=96 + tensor_model_parallel_size=8 + ;; + 22) + model_name=gpt_22B + num_layers=48 + hidden_size=6144 + num_attention_heads=64 + tensor_model_parallel_size=8 + ;; + 13) + model_name=gpt_13B + num_layers=40 + hidden_size=5120 + num_attention_heads=40 + ;; + 7) + model_name=gpt_7B + num_layers=36 + hidden_size=4096 + num_attention_heads=32 + tensor_model_parallel_size=4 + ;; + 405) + model_name=llama_405B + num_layers=128 + hidden_size=16384 + ffn_hidden_size=53248 + num_attention_heads=128 + ;; + moe) + model_name=Mixtral_8*7B + num_layers=32 + hidden_size=4096 + num_attention_heads=32 + ffn_hidden_size=14336 + tensor_model_parallel_size=4 + moe_enable=--moe_enable + grouped_gemm=--moe_grouped_gemm + ;; + deepseek671) + model_name=DeepSeek_671B + frame=DeepSeek + moe_enable=--moe_enable + num_layers=61 + hidden_size=18432 + num_attention_heads=128 + ffn_hidden_size=2048 + q_lora_rank=1536 + kv_lora_rank=512 + qk_nope_dim=128 + qk_rope_dim=64 + v_head_dim=128 + num_experts=256 + n_shared_expert=1 + n_dense_layer=3 + tensor_model_parallel_size=1 + ;; + deepseek236) + model_name=DeepSeek_236B + frame=DeepSeek + moe_enable=--moe_enable + num_layers=60 + hidden_size=12288 + num_attention_heads=128 + ffn_hidden_size=1536 + q_lora_rank=1536 + kv_lora_rank=512 + qk_nope_dim=128 + qk_rope_dim=64 + v_head_dim=128 + num_experts=160 + n_shared_expert=2 + n_dense_layer=1 + tensor_model_parallel_size=1 + ;; + deepseek16) + model_name=DeepSeek_16B + frame=DeepSeek + moe_enable=--moe_enable + num_layers=27 + hidden_size=10944 + num_attention_heads=16 + ffn_hidden_size=1408 + q_lora_rank=0 + kv_lora_rank=512 + qk_nope_dim=128 + qk_rope_dim=64 + v_head_dim=128 + num_experts=64 + n_shared_expert=2 + n_dense_layer=1 + tensor_model_parallel_size=1 + ;; + deepseek) + frame=DeepSeek + model_name="DeepSeek" + moe_enable=--moe_enable + ;; + # set everything else from cmdline + # or it defaults to utils/utils.py's arg parser + (*) + echo "Only support model size 175, 22,13 or 7; using default size 13" + model_name=gpt_13B + num_layers=40 + hidden_size=5120 + num_attention_heads=40 + ;; +esac + + +cmd="python -m workload_generator.SimAI_training_workload_generator \ + --gpu_type=$gpu_type \ + --frame=$frame \ + --world_size=$world_size \ + --tensor_model_parallel_size=$tensor_model_parallel_size \ + --pipeline_model_parallel=$pipeline_model_parallel \ + --global_batch=$global_batch \ + --micro_batch=$micro_batch \ + --num_layers=$num_layers \ + --seq_length=$seq_length \ + --hidden_size=$hidden_size \ + --epoch_num=$epoch_num \ + --num_attention_heads=$num_attention_heads \ + --model_name=$model_name \ + --max_position_embeddings=$max_position_embeddings \ + --vocab_size=$vocab_size \ + --use-distributed-optimizer + ${aiob_enable} \ + ${use_flash_attn} \ + ${swiglu} \ + ${sp_enable} \ + ${recompute_activations} \ + ${ffn_hidden_size:+--ffn_hidden_size=$ffn_hidden_size} \ + ${comp_filepath:+--comp_filepath=$comp_filepath} \ + ${moe_enable} \ + ${moe_router_topk:+--moe_router_topk=$moe_router_topk} \ + ${num_experts:+--num_experts=$num_experts} \ + ${expert_model_parallel_size:+--expert_model_parallel_size=$expert_model_parallel_size} \ + ${grouped_gemm} \ + ${q_lora_rank:+--q_lora_rank=$q_lora_rank} \ + ${kv_lora_rank:+--kv_lora_rank=$kv_lora_rank} \ + ${qk_nope_dim:+--qk_nope_dim=$qk_nope_dim} \ + ${qk_rope_dim:+--qk_rope_dim=$qk_rope_dim} \ + ${v_head_dim:+--v_head_dim=$v_head_dim} \ + ${n_shared_expert:+--n_shared_expert=$n_shared_expert} \ + ${n_dense_layer:+--n_dense_layer=$n_dense_layer} \ + " \ + +echo $cmd + + +$cmd diff --git a/aicb/scripts/merge_per_rank_csv.py b/aicb/scripts/merge_per_rank_csv.py new file mode 100644 index 000000000..01c291c8d --- /dev/null +++ b/aicb/scripts/merge_per_rank_csv.py @@ -0,0 +1,127 @@ +"""Merge per-rank CSVs into a single file. Only the ranks column varies — +instead of one file per rank, output one file where each row shows ALL groups. + +Usage: + cd aicb && python3 scripts/merge_per_rank_csv.py \ + --csv results/mocked_workload/megatron_xxx_workload.csv +""" + +import argparse, csv, os, sys +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from utils.utils import RankGenerator, CommGroup +from utils.rank_mapper import _COMM_GROUP_TOKEN_MAP + + +def _detect_from_sidecar(sidecar_path): + if not os.path.exists(sidecar_path): + return {} + info = {} + with open(sidecar_path, newline='') as f: + for row in csv.reader(f): + if not row or row[0] == 'group': + continue + name, size_str = row[0], row[1] + size = int(size_str) + if name == 'all_nodes': info['world_size'] = size + elif name == 'tp_group': info['tp'] = size + elif name == 'pp_group': info['pp'] = size + elif name == 'dp_group': info['dp_full'] = size + elif name == 'ep_group': + rank_groups = row[2].split() + info['ep'] = size if len(rank_groups) > 0 and size > 1 else 1 + return info + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--csv", required=True) + parser.add_argument("--tp", type=int, default=None) + parser.add_argument("--pp", type=int, default=None) + parser.add_argument("--ep", type=int, default=None) + parser.add_argument("--cp", type=int, default=1) + parser.add_argument("--order", default="tp-cp-ep-dp-pp") + parser.add_argument("--out", default=None) + args = parser.parse_args() + + sidecar = args.csv.replace('_workload.csv', '_rank_mapping.csv') + info = _detect_from_sidecar(sidecar) + world_size = info.get('world_size') + tp = args.tp or info.get('tp', 1) + pp = args.pp or info.get('pp', 1) + ep = args.ep or info.get('ep', 1) + if world_size is None: + world_size = max(tp * pp, 1) + dp = world_size // (tp * pp) + + rg = RankGenerator(tp=tp, ep=ep, dp=dp, pp=pp, cp=args.cp, order=args.order) + + # Read the base CSV (rank 0) + with open(args.csv, newline='') as f: + reader = csv.reader(f) + all_rows = list(reader) + + header = all_rows[0] + data_rows = all_rows[1:] + ranks_col = len(header) - 1 + + # Replace 'ranks' header with 'all_groups' + new_header = list(header) + new_header[ranks_col] = 'all_groups' + + out_file = args.out or args.csv.replace('_workload.csv', '_merged.csv') + with open(out_file, 'w', newline='') as f: + writer = csv.writer(f) + writer.writerow(new_header) + + for row in data_rows: + cg_raw = row[1] if len(row) > 1 else '' + new_row = list(row) + while len(new_row) <= ranks_col: + new_row.append('') + + if cg_raw in ('None', '', 'CommGroup.None'): + new_row[ranks_col] = '' + else: + cg_name = cg_raw.split('.')[-1] + try: + cg = CommGroup(cg_name) + except ValueError: + new_row[ranks_col] = '' + writer.writerow(new_row) + continue + + # Get ALL groups for this CommGroup type + if cg == CommGroup.all: + all_groups_str = f"[{','.join(str(r) for r in range(world_size))}]" + elif cg in _COMM_GROUP_TOKEN_MAP: + token, indep = _COMM_GROUP_TOKEN_MAP[cg] + groups = rg.get_ranks(token, independent_ep=indep) + all_groups_str = ' '.join( + '[' + ','.join(str(r) for r in g) + ']' for g in groups + ) + else: + all_groups_str = '' + + new_row[ranks_col] = '"' + all_groups_str + '"' + + writer.writerow(new_row) + + print(f"Merged CSV: {out_file}") + print(f" rows={len(data_rows)}, groups_col shows all {world_size}-rank decomposition") + + # Preview + with open(out_file, newline='') as f: + rows_out = list(csv.reader(f)) + print(f"\n=== Preview (first 5 comm rows) ===") + count = 0 + for row in rows_out: + if row[ranks_col] and row[ranks_col] != '""': + display = row[ranks_col][:150] + ('...' if len(row[ranks_col]) > 150 else '') + print(f" {row[4][:35]:<35} {row[1].split('.')[-1]:<15} {display}") + count += 1 + if count >= 5: + break + + +if __name__ == "__main__": + main() diff --git a/aicb/scripts/run_in_cluster.py b/aicb/scripts/run_in_cluster.py new file mode 100644 index 000000000..c8487b5ce --- /dev/null +++ b/aicb/scripts/run_in_cluster.py @@ -0,0 +1,63 @@ +#!/usr/bin/python3 +""" +Usage [{filename}]: +1. Change IMAGE_NAME from DUMMY to your real image name +2. Change IPLIST from DUMMY to your real /path/to/iplist (absolute path) +3. Change AICB_DIR from DUMMY to your real /path/to/aicb (absolute path) +4. Change the settings in run_suites.py to select the workload you want +5. Copy iplist and aicb to all participating servers at /path/to/iplist and /path/to/aicb, e.g., using `pscp` command like `pscp.pssh -h iplist iplist /path/to/iplist` and `pscp.pssh -h iplist -r aicb /path/to/aicb` +6. Run simulation on all participating servers, e.g., using `pssh` command like `pssh -i -h /path/to/iplist -o out -e err -t 0 "cd /path/to/aicb && python scripts/run_in_cluster.py"` +""" + +import subprocess +import os +import re +import sys + +filename = os.path.basename(__file__) +__doc__ = __doc__.format(filename=filename) + + +def get_local_ip(): + output = os.popen("ifconfig").read().strip() + pattern = r"inet (\d+.\d+.\d+.\d+) " + return re.findall(pattern, output) + + +def get_world_id_list(filename): + with open(filename, "r") as f: + return f.read().strip().split("\n") + + +def get_docker_env_rank(filename): + ip_list = get_world_id_list(filename) + local_ip = get_local_ip() + for ip in local_ip: + if ip in ip_list: + return len(ip_list), ip_list.index(ip), ip_list[0], 12345 + return -1, -1, -1, -1 + + +IPLIST = "DUMMY_IPLIST" # Change it to /path/to/iplist, e.g., /root/iplist +AICB_DIR = "DUMMY_AICB_DIR" # Change it to /path/to/aicb, e.g., /root/aicb +IMAGE_NAME = "DUMMY_IMAGE_NAME" # Change it to your docker image name, e.g., nvcr.io/nvidia/pytorch:xx.xx-py3 + +if IPLIST == "DUMMY_IPLIST" or AICB_DIR == "DUMMY_AICB_DIR" or IMAGE_NAME == "DUMMY_IMAGE_NAME": + sys.stderr.write(__doc__) + sys.exit(1) + +WORLD_SIZE, RANK, MASTER_ADDR, MASTER_PORT = get_docker_env_rank(IPLIST) +AICB_DIR_base = os.path.basename(AICB_DIR) +command = f"""docker run --name aicb_test --gpus all --privileged \ +--ulimit memlock=-1 --ulimit stack=67108864 \ +--init -i --shm-size=4g --network=host --rm \ +-e WORLD_SIZE={WORLD_SIZE} \ +-e RANK={RANK} \ +-e MASTER_ADDR={MASTER_ADDR} \ +-e MASTER_PORT={MASTER_PORT} \ +-v {AICB_DIR}:/workspace/{AICB_DIR_base} \ +{IMAGE_NAME} /bin/sh -c 'cd /workspace/{AICB_DIR_base} && pwd && python run_suites.py' +""" # Change the settings in run_suites.py to select the workload you want + +ret = subprocess.run(command, shell=True) +print(ret) diff --git a/aicb/tests/__init__.py b/aicb/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/aicb/tests/test_logitem_ranks.py b/aicb/tests/test_logitem_ranks.py new file mode 100644 index 000000000..bd0332fa4 --- /dev/null +++ b/aicb/tests/test_logitem_ranks.py @@ -0,0 +1,26 @@ +"""Unit tests for LogItem ranks field serialization.""" +from log_analyzer.log import LogItem +from utils.utils import CommType, CommGroup + +class TestLogItemRanks: + def test_ranks_list_to_csv(self): + li = LogItem(comm_type=CommType.all_reduce, ranks=[0, 1, 2, 3]) + assert '"0,1,2,3"' in li.view_as_csv_line() + + def test_ranks_none_to_csv(self): + li = LogItem(comm_type=CommType.all_reduce, ranks=None) + assert li.view_as_csv_line().endswith(',') + + def test_ranks_empty_to_csv(self): + li = LogItem(comm_type=CommType.all_reduce, ranks=[]) + assert '""' in li.view_as_csv_line() + + def test_ranks_single_to_csv(self): + li = LogItem(comm_type=CommType.all_reduce, ranks=[0]) + assert '"0"' in li.view_as_csv_line() + + def test_ranks_in_header(self): + assert 'ranks' in LogItem(comm_type=CommType.all_reduce).csv_header() + + def test_default_ranks_is_none(self): + assert LogItem(comm_type=CommType.all_reduce).ranks is None diff --git a/aicb/tests/test_mocked_qwen3.py b/aicb/tests/test_mocked_qwen3.py new file mode 100644 index 000000000..e15660f72 --- /dev/null +++ b/aicb/tests/test_mocked_qwen3.py @@ -0,0 +1,504 @@ +"""Parametrized communication-count tests for MockedQwen3 (dense + MoE).""" + +import types +import pytest + +from utils.utils import CommType + + +# --------------------------------------------------------------------------- +# Helper: build a mock argparse namespace for MegatronWorkload +# --------------------------------------------------------------------------- + +def _make_args(hidden_size, num_layers, tp=4, ep=1, dp=2, seq=4096, batch=2, moe=False): + a = types.SimpleNamespace() + a.frame = "Qwen3" + a.model_name = "test" + a.hidden_size = hidden_size + a.num_hidden_layers = num_layers + a.ffn_hidden_size = 0 + a.vocab_size = 151936 + a.tensor_model_parallel_size = tp + a.pipeline_model_parallel = 1 + a.world_size = tp * ep * dp + a.dp_num = dp + a.expert_model_parallel_size = ep + a.context_parallel_size = 1 + a.seq_length = seq + a.micro_batch = batch + a.global_batch = 64 + a.epoch_num = 1 + a.num_microbatches = 1 + a.num_attention_heads = 0 + a.num_key_value_heads = 0 + a.head_dim = 128 + a.enable_sequence_parallel = True + a.computation_enable = False + a.moe_enable = moe + a.swiglu = True + a.add_bias_linear = False + a.use_distributed_optimizer = True + a.pp_rank = 0 + a.workload_only = True + a.order = "tp-cp-ep-dp-pp" + a.aiob_enable = False + a.padded_vocab_size = 151936 + return a + + +# --------------------------------------------------------------------------- +# Model configs (verified from HF config.json) +# --------------------------------------------------------------------------- + +QWEN3_DENSE = [ + # name hidden intermed layers Q_heads KV_heads head_dim tie_emb + ("Qwen3-0.6B", 1024, 3072, 28, 16, 8, 128, True), + ("Qwen3-1.7B", 2048, 6144, 28, 16, 8, 128, True), + ("Qwen3-4B", 2560, 9728, 36, 32, 8, 128, True), + ("Qwen3-8B", 4096, 12288, 36, 32, 8, 128, False), + ("Qwen3-14B", 5120, 17408, 40, 40, 8, 128, False), + ("Qwen3-32B", 5120, 25600, 64, 64, 8, 128, False), +] + +QWEN3_MOE = [ + # name hidden intermed layers Q_heads KV_heads head_dim moe_ff topk experts shared + ("Qwen3-30B-A3B", 2048, 6144, 48, 32, 4, 128, 768, 8, 128, False), + ("Qwen3-235B-A22B", 4096, 12288, 94, 64, 4, 128, 1536, 8, 128, False), +] + + +# --------------------------------------------------------------------------- +# Dense model comm-count tests +# --------------------------------------------------------------------------- + +class TestQwen3DenseCommCounts: + """Per-layer comm-ops formula for all 6 Qwen3 dense sizes at TP=4, SP=True.""" + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,tie", QWEN3_DENSE) + def test_allgather_count(self, name, h, ff, L, n_heads, n_kv, hd, tie): + from workload_generator.mocked_model.training.MockedQwen3 import ( + Qwen3Params, Qwen3Model, + ) + from workload_generator.generate_megatron_workload import MegatronWorkload + + cfg = _build_dense_cfg(h, ff, L, n_heads, n_kv, hd, tie) + model = Qwen3Model(cfg) + args = _make_args(h, L) + wl = MegatronWorkload(args, model)() + + ag = sum(1 for w in wl.workload if str(w.comm_type) == "CommType.all_gather") + expected = L * 6 + 2 + 3 # layers + lm_head + init/step + assert ag == expected, f"{name}: all_gather {ag} != {expected}" + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,tie", QWEN3_DENSE) + def test_reduce_scatter_count(self, name, h, ff, L, n_heads, n_kv, hd, tie): + from workload_generator.mocked_model.training.MockedQwen3 import ( + Qwen3Params, Qwen3Model, + ) + from workload_generator.generate_megatron_workload import MegatronWorkload + + cfg = _build_dense_cfg(h, ff, L, n_heads, n_kv, hd, tie) + model = Qwen3Model(cfg) + args = _make_args(h, L) + wl = MegatronWorkload(args, model)() + + rs = sum(1 for w in wl.workload if str(w.comm_type) == "CommType.reduce_scatter") + expected = L * 4 + 1 + 1 # layers + lm_head + step + assert rs == expected, f"{name}: reduce_scatter {rs} != {expected}" + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,tie", QWEN3_DENSE) + def test_message_size(self, name, h, ff, L, n_heads, n_kv, hd, tie): + from workload_generator.mocked_model.training.MockedQwen3 import ( + Qwen3Params, Qwen3Model, + ) + from workload_generator.generate_megatron_workload import MegatronWorkload + + cfg = _build_dense_cfg(h, ff, L, n_heads, n_kv, hd, tie) + model = Qwen3Model(cfg) + args = _make_args(h, L) + wl = MegatronWorkload(args, model)() + + exp_msg = 2 * 4096 * 2 * h # 2 * seq * batch * hidden_size + for w in wl.workload: + if str(w.comm_type) == "CommType.all_gather" and "ColumnLinear" in (w.stage or ""): + assert w.msg_size == exp_msg, f"{name}: ColumnLinear AG msg {w.msg_size} != {exp_msg}" + if str(w.comm_type) == "CommType.reduce_scatter" and "RowLinear" in (w.stage or ""): + assert w.msg_size == exp_msg, f"{name}: RowLinear RS msg {w.msg_size} != {exp_msg}" + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,tie", QWEN3_DENSE) + def test_qk_norm_params(self, name, h, ff, L, n_heads, n_kv, hd, tie): + from workload_generator.mocked_model.training.MockedQwen3 import ( + Qwen3Params, Qwen3Model, + ) + + cfg = _build_dense_cfg(h, ff, L, n_heads, n_kv, hd, tie) + model = Qwen3Model(cfg) + + qk_params = [p for p in model.parameters() if "q_norm" in (p.name or "")] + assert len(qk_params) == L, f"{name}: q_norm count {len(qk_params)} != {L}" + for p in qk_params: + assert p.numel() == 128, f"{name}: q_norm dim {p.numel()} != 128" + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,tie", QWEN3_DENSE) + def test_tie_word_embeddings(self, name, h, ff, L, n_heads, n_kv, hd, tie): + from workload_generator.mocked_model.training.MockedQwen3 import ( + Qwen3Params, Qwen3Model, + ) + + cfg = _build_dense_cfg(h, ff, L, n_heads, n_kv, hd, tie) + model = Qwen3Model(cfg) + + lm_params = sum(p.numel() for p in model.lm_head.parameters()) + if tie: + assert lm_params == 0, f"{name}: tied lm_head should have 0 params, got {lm_params}" + else: + expected = 151936 * h // 4 # vocab * hidden / TP + assert lm_params == expected, f"{name}: lm_head params {lm_params} != {expected}" + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,tie", QWEN3_DENSE) + def test_embedding_no_megatron_artifacts(self, name, h, ff, L, n_heads, n_kv, hd, tie): + from workload_generator.mocked_model.training.MockedQwen3 import ( + Qwen3Params, Qwen3Model, + ) + + cfg = _build_dense_cfg(h, ff, L, n_heads, n_kv, hd, tie) + model = Qwen3Model(cfg) + + emb_params = sum(p.numel() for p in model.embedding.parameters()) + expected = 151936 * h // 4 # vocab * hidden / TP (no 4x multiplier) + assert emb_params == expected, ( + f"{name}: embedding {emb_params} != {expected} " + f"(ratio={emb_params/expected:.2f}x, should be 1.0x)" + ) + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,tie", QWEN3_DENSE) + def test_head_expansion(self, name, h, ff, L, n_heads, n_kv, hd, tie): + from workload_generator.mocked_model.training.MockedQwen3 import ( + Qwen3Params, Qwen3Model, + ) + + cfg = _build_dense_cfg(h, ff, L, n_heads, n_kv, hd, tie) + model = Qwen3Model(cfg) + + attn = model.layers[0].attention + expected_q = n_heads * hd + assert attn.query_projection_size == expected_q, ( + f"{name}: Q dim {attn.query_projection_size} != {expected_q}" + ) + expected_kv = n_kv * hd + assert attn.kv_projection_size == expected_kv, ( + f"{name}: KV dim {attn.kv_projection_size} != {expected_kv}" + ) + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,tie", QWEN3_DENSE) + def test_qk_norm_zero_communication(self, name, h, ff, L, n_heads, n_kv, hd, tie): + from workload_generator.mocked_model.training.MockedQwen3 import ( + Qwen3Params, Qwen3Model, + ) + from workload_generator.generate_megatron_workload import MegatronWorkload + + cfg = _build_dense_cfg(h, ff, L, n_heads, n_kv, hd, tie) + model = Qwen3Model(cfg) + args = _make_args(h, L) + wl = MegatronWorkload(args, model)() + + qk_items = [w for w in wl.workload + if w.stage and ("q_norm" in w.stage or "k_norm" in w.stage)] + assert len(qk_items) == 0, f"{name}: QK-Norm should produce 0 comm items, got {len(qk_items)}" + + +# --------------------------------------------------------------------------- +# MoE model comm-count tests +# --------------------------------------------------------------------------- + +class TestQwen3MoeCommCounts: + """Per-layer comm-ops formula for Qwen3 MoE models at TP=4, EP=4. + + The rank-mapper crashes with ep_group present (ZeroDivisionError in + _fill_ranks). This is a pre-existing upstream issue, not related to + Qwen3 mocks. The fixture patches WorkloadGenerator.__call__ to skip + rank-filling so communication counts can be verified. + """ + + @pytest.fixture(autouse=True) + def _skip_rank_fill(self, monkeypatch): + """Patch WorkloadGenerator.__call__ to skip _fill_ranks.""" + from workload_generator.workload_generator import WorkloadGenerator + + def _call_without_fill(self): + args = self.args + from log_analyzer.log import Workload, LogItem + from utils.utils import CommType + self.workload = Workload() + self.init() + self.workload.append(LogItem(comm_type=CommType.epoch_end)) + for _ in range(args.epoch_num): + if args.pipeline_model_parallel > 1 and args.frame != "collective_test": + self.with_pipeline_forward_backward() + else: + for _ in range(args.num_microbatches): + self.forward() + self.backward() + self.step() + self.workload.append(LogItem(comm_type=CommType.epoch_end)) + return self.workload + + monkeypatch.setattr(WorkloadGenerator, "__call__", _call_without_fill) + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,moe_ff,topk,nexp,shared", QWEN3_MOE) + def test_alltoall_count(self, name, h, ff, L, n_heads, n_kv, hd, moe_ff, topk, nexp, shared): + from workload_generator.mocked_model.training.MockedQwen3 import ( + Qwen3Params, Qwen3Model, + ) + from workload_generator.generate_megatron_workload import MegatronWorkload + + cfg = _build_moe_cfg(h, ff, L, n_heads, n_kv, hd, moe_ff, topk, nexp) + model = Qwen3Model(cfg) + args = _make_args(h, L, ep=4, dp=2, moe=True) + wl = MegatronWorkload(args, model)() + + a2a = sum(1 for w in wl.workload if str(w.comm_type) == "CommType.all_to_all") + expected = L * 4 # 4 A2A per MoE layer (fwd dispatch+combine, bwd dispatch+combine) + assert a2a == expected, f"{name}: all_to_all {a2a} != {expected}" + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,moe_ff,topk,nexp,shared", QWEN3_MOE) + def test_alltoall_symmetry(self, name, h, ff, L, n_heads, n_kv, hd, moe_ff, topk, nexp, shared): + from workload_generator.mocked_model.training.MockedQwen3 import ( + Qwen3Params, Qwen3Model, + ) + from workload_generator.generate_megatron_workload import MegatronWorkload + + cfg = _build_moe_cfg(h, ff, L, n_heads, n_kv, hd, moe_ff, topk, nexp) + model = Qwen3Model(cfg) + args = _make_args(h, L, ep=4, dp=2, moe=True) + wl = MegatronWorkload(args, model)() + + moe_items = [w for w in wl.workload if w.stage and "MoE" in w.stage] + fwd_a2a = sum(1 for w in moe_items + if w.stage.startswith("forward") and str(w.comm_type) == "CommType.all_to_all") + bwd_a2a = sum(1 for w in moe_items + if w.stage.startswith("backward") and str(w.comm_type) == "CommType.all_to_all") + assert fwd_a2a == bwd_a2a, ( + f"{name}: A2A asymmetry fwd={fwd_a2a} bwd={bwd_a2a}" + ) + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,moe_ff,topk,nexp,shared", QWEN3_MOE) + def test_alltoall_message_size(self, name, h, ff, L, n_heads, n_kv, hd, moe_ff, topk, nexp, shared): + from workload_generator.mocked_model.training.MockedQwen3 import ( + Qwen3Params, Qwen3Model, + ) + from workload_generator.generate_megatron_workload import MegatronWorkload + + cfg = _build_moe_cfg(h, ff, L, n_heads, n_kv, hd, moe_ff, topk, nexp) + model = Qwen3Model(cfg) + args = _make_args(h, L, ep=4, dp=2, moe=True) + wl = MegatronWorkload(args, model)() + + exp_msg = 4096 * h * 2 * topk // 4 // 4 * 2 # seq * hidden * batch * topk // tp // ep * 2 + for w in wl.workload: + if str(w.comm_type) == "CommType.all_to_all": + assert w.msg_size == exp_msg, ( + f"{name}: A2A msg {w.msg_size} != {exp_msg}" + ) + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,moe_ff,topk,nexp,shared", QWEN3_MOE) + def test_moe_backward_not_empty(self, name, h, ff, L, n_heads, n_kv, hd, moe_ff, topk, nexp, shared): + from workload_generator.mocked_model.training.MockedQwen3 import ( + Qwen3Params, Qwen3Model, + ) + from workload_generator.generate_megatron_workload import MegatronWorkload + + cfg = _build_moe_cfg(h, ff, L, n_heads, n_kv, hd, moe_ff, topk, nexp) + model = Qwen3Model(cfg) + args = _make_args(h, L, ep=4, dp=2, moe=True) + wl = MegatronWorkload(args, model)() + + bwd_moe = [w for w in wl.workload + if w.stage and w.stage.startswith("backward") and "MoE" in w.stage] + assert len(bwd_moe) > 0, ( + f"{name}: MoE backward is empty! MOEMLP backward fix may not be applied." + ) + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,moe_ff,topk,nexp,shared", QWEN3_MOE) + def test_moe_reduce_scatter_count(self, name, h, ff, L, n_heads, n_kv, hd, moe_ff, topk, nexp, shared): + from workload_generator.mocked_model.training.MockedQwen3 import ( + Qwen3Params, Qwen3Model, + ) + from workload_generator.generate_megatron_workload import MegatronWorkload + + cfg = _build_moe_cfg(h, ff, L, n_heads, n_kv, hd, moe_ff, topk, nexp) + model = Qwen3Model(cfg) + args = _make_args(h, L, ep=4, dp=2, moe=True) + wl = MegatronWorkload(args, model)() + + rs = sum(1 for w in wl.workload if str(w.comm_type) == "CommType.reduce_scatter") + expected = L * 4 + 2 # layers + lm_head + step + assert rs == expected, f"{name}: reduce_scatter {rs} != {expected}" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _build_dense_cfg(hidden_size, intermediate_size, num_layers, + num_heads, num_kv, head_dim, tie_emb): + cfg = types.SimpleNamespace() + cfg.hidden_size = hidden_size + cfg.intermediate_size = intermediate_size + cfg.num_hidden_layers = num_layers + cfg.num_attention_heads = num_heads + cfg.num_key_value_heads = num_kv + cfg.head_dim = head_dim + cfg.vocab_size = 151936 + cfg.tie_word_embeddings = tie_emb + cfg.tensor_model_parallel_size = 4 + cfg.world_size = 8 + cfg.seq_length = 4096 + cfg.micro_batch = 2 + cfg.enable_sequence_parallel = True + cfg.computation_enable = False + cfg.add_bias_linear = False + cfg.moe_enable = False + cfg.moe_intermediate_size = 0 + cfg.moe_router_topk = 0 + cfg.num_experts = 0 + cfg.expert_model_parallel_size = 1 + return cfg + + +def _build_moe_cfg(hidden_size, intermediate_size, num_layers, + num_heads, num_kv, head_dim, moe_ff, topk, num_experts): + cfg = types.SimpleNamespace() + cfg.hidden_size = hidden_size + cfg.intermediate_size = intermediate_size + cfg.num_hidden_layers = num_layers + cfg.num_attention_heads = num_heads + cfg.num_key_value_heads = num_kv + cfg.head_dim = head_dim + cfg.vocab_size = 151936 + cfg.tie_word_embeddings = False + cfg.tensor_model_parallel_size = 4 + cfg.expert_model_parallel_size = 4 + cfg.world_size = 32 + cfg.seq_length = 4096 + cfg.micro_batch = 2 + cfg.enable_sequence_parallel = True + cfg.computation_enable = False + cfg.add_bias_linear = False + cfg.moe_enable = True + cfg.moe_intermediate_size = moe_ff + cfg.moe_router_topk = topk + cfg.num_experts = num_experts + return cfg + +# --------------------------------------------------------------------------- +# SimAI training workload generator (.txt format for C++ simulator) +# --------------------------------------------------------------------------- + +class TestSimAITrainingGenerator: + """End-to-end: Qwen3Model -> SIMAI_workload -> .txt (HYBRID_TRANSFORMER).""" + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,tie", QWEN3_DENSE) + def test_txt_format_hybrid_transformer_header(self, name, h, ff, L, n_heads, n_kv, hd, tie): + from workload_generator.mocked_model.training.MockedQwen3 import Qwen3Model + from workload_generator.SimAI_training_workload_generator import SIMAI_workload + import tempfile, os + + cfg = _build_dense_cfg(h, ff, L, n_heads, n_kv, hd, tie) + model = Qwen3Model(cfg) + args = _make_simai_args(h, L) + work = SIMAI_workload(model, args, None) + work.workload_generate() + with tempfile.NamedTemporaryFile(suffix=".txt", delete=False, prefix="t_") as tmp: + txt_path = tmp.name + try: + work.dump_file(txt_path.replace(".txt", "")) + with open(txt_path) as f: + first = f.readline().strip() + assert "HYBRID_TRANSFORMER" in first, f"{name}: missing header" + finally: + os.unlink(txt_path) + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,tie", QWEN3_DENSE) + def test_txt_has_required_comm_types(self, name, h, ff, L, n_heads, n_kv, hd, tie): + from workload_generator.mocked_model.training.MockedQwen3 import Qwen3Model + from workload_generator.SimAI_training_workload_generator import SIMAI_workload + import tempfile, os + + cfg = _build_dense_cfg(h, ff, L, n_heads, n_kv, hd, tie) + model = Qwen3Model(cfg) + args = _make_simai_args(h, L) + work = SIMAI_workload(model, args, None) + work.workload_generate() + with tempfile.NamedTemporaryFile(suffix=".txt", delete=False, prefix="t_") as tmp: + txt_path = tmp.name + try: + work.dump_file(txt_path.replace(".txt", "")) + with open(txt_path) as f: + lines = f.read().strip().split("\n") + comm_types = set() + for i in range(2, len(lines)): + parts = lines[i].split("\t") + if len(parts) >= 7: + comm_types.add(parts[3]); comm_types.add(parts[6]) + assert "ALLGATHER" in comm_types, f"{name}: missing ALLGATHER" + assert "REDUCESCATTER" in comm_types, f"{name}: missing REDUCESCATTER" + assert "ALLREDUCE" in comm_types, f"{name}: missing ALLREDUCE" + finally: + os.unlink(txt_path) + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,tie", QWEN3_DENSE) + def test_txt_message_sizes_match(self, name, h, ff, L, n_heads, n_kv, hd, tie): + from workload_generator.mocked_model.training.MockedQwen3 import Qwen3Model + from workload_generator.SimAI_training_workload_generator import SIMAI_workload + import tempfile, os + + cfg = _build_dense_cfg(h, ff, L, n_heads, n_kv, hd, tie) + model = Qwen3Model(cfg) + args = _make_simai_args(h, L) + work = SIMAI_workload(model, args, None) + work.workload_generate() + with tempfile.NamedTemporaryFile(suffix=".txt", delete=False, prefix="t_") as tmp: + txt_path = tmp.name + try: + work.dump_file(txt_path.replace(".txt", "")) + with open(txt_path) as f: + lines = f.read().strip().split("\n") + exp_tp = 2 * 4096 * 2 * h + found = any( + line.split("\t")[3] in ("ALLGATHER", "REDUCESCATTER") + and int(line.split("\t")[4]) == exp_tp + for line in lines[2:] + if len(line.split("\t")) >= 5 + ) + assert found, f"{name}: TP comm size {exp_tp} not found" + finally: + os.unlink(txt_path) + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,tie", QWEN3_DENSE) + def test_txt_produces_nonempty_workload(self, name, h, ff, L, n_heads, n_kv, hd, tie): + from workload_generator.mocked_model.training.MockedQwen3 import Qwen3Model + from workload_generator.SimAI_training_workload_generator import SIMAI_workload + + cfg = _build_dense_cfg(h, ff, L, n_heads, n_kv, hd, tie) + model = Qwen3Model(cfg) + args = _make_simai_args(h, L) + work = SIMAI_workload(model, args, None) + work.workload_generate() + assert len(work.workload) > 0, f"{name}: empty workload" + + +def _make_simai_args(hidden_size, num_layers): + a = types.SimpleNamespace() + a.frame = "Qwen3"; a.model_name = "test"; a.hidden_size = hidden_size + a.num_hidden_layers = num_layers; a.ffn_hidden_size = 0; a.vocab_size = 151936 + a.tensor_model_parallel_size = 4; a.pipeline_model_parallel = 1 + a.world_size = 8; a.dp_num = 2; a.expert_model_parallel_size = 1 + a.seq_length = 4096; a.micro_batch = 2; a.global_batch = 64 + a.enable_sequence_parallel = True; a.moe_enable = False + a.moe_router_topk = 0; a.num_experts = 0; a.moe_grouped_gemm = True + a.use_flash_attn = False; a.gpu_type = "A100"; a.ga_num = 8 + a.recompute_activations = False; a.aiob_enable = False + a.num_layers = num_layers; a.epoch_num = 1 + a.num_attention_heads = 0; a.num_key_value_heads = 0; a.head_dim = 128 + return a diff --git a/aicb/tests/test_mocked_qwen3_5.py b/aicb/tests/test_mocked_qwen3_5.py new file mode 100644 index 000000000..e6c11ce41 --- /dev/null +++ b/aicb/tests/test_mocked_qwen3_5.py @@ -0,0 +1,414 @@ +"""Parametrized communication-count tests for MockedQwen3_5 (dense + MoE).""" + +import types +import pytest + +from utils.utils import CommType + + +# --------------------------------------------------------------------------- +# Qwen3.5 model configs (verified from HF config.json) +# --------------------------------------------------------------------------- + +QWEN35_DENSE = [ + # name h ff L Q_heads KV_heads hd tie linear_khd linear_vhd + ("Qwen3.5-0.8B", 1024, 3584, 24, 8, 2, 256, True, 128, 16), + ("Qwen3.5-2B", 2048, 6144, 24, 8, 2, 256, True, 128, 16), + ("Qwen3.5-4B", 2560, 9216, 32, 16, 4, 256, True, 128, 32), + ("Qwen3.5-9B", 4096, 12288, 32, 16, 4, 256, False, 128, 32), + ("Qwen3.5-27B", 5120, 17408, 64, 24, 4, 256, False, 128, 48), +] + +QWEN35_MOE = [ + # name h ff L Q_heads KV_heads hd moe_ff topk experts shared + ("Qwen3.5-35B-A3B", 2048, 6144, 40, 16, 2, 256, 512, 8, 256, 512), + ("Qwen3.5-122B-A10B",3072, 9216, 48, 32, 2, 256, 1024, 8, 256, 1024), + ("Qwen3.5-397B-A17B",4096,12288,60, 32, 2, 256, 1024, 10, 512, 1024), +] + + +def _make_args35(hidden_size, num_layers, tp=4, ep=1, dp=2, seq=4096, batch=2, moe=False): + a = types.SimpleNamespace() + a.frame = "Qwen3.5"; a.model_name = "test"; a.hidden_size = hidden_size + a.num_hidden_layers = num_layers; a.ffn_hidden_size = 0; a.vocab_size = 248320 + a.tensor_model_parallel_size = tp; a.pipeline_model_parallel = 1 + a.world_size = tp * ep * dp; a.dp_num = dp + a.expert_model_parallel_size = ep; a.context_parallel_size = 1 + a.seq_length = seq; a.micro_batch = batch; a.global_batch = 64 + a.epoch_num = 1; a.num_microbatches = 1 + a.num_attention_heads = 0; a.num_key_value_heads = 0; a.head_dim = 256 + a.enable_sequence_parallel = True; a.computation_enable = False + a.moe_enable = moe; a.swiglu = True; a.add_bias_linear = False + a.use_distributed_optimizer = True; a.pp_rank = 0 + a.workload_only = True; a.order = "tp-cp-ep-dp-pp"; a.aiob_enable = False + a.padded_vocab_size = 248320; a.recompute_activations = False + a.num_layers = num_layers; a.ga_num = 8; a.gpu_type = "A100" + a.use_flash_attn = False + return a + + +def _build_dense_cfg35(hidden_size, intermediate_size, num_layers, + num_heads, num_kv, head_dim, tie_emb, + linear_key_dim, linear_value_dim): + cfg = types.SimpleNamespace() + cfg.hidden_size = hidden_size + cfg.intermediate_size = intermediate_size + cfg.num_hidden_layers = num_layers + cfg.num_attention_heads = num_heads + cfg.num_key_value_heads = num_kv + cfg.head_dim = head_dim + cfg.vocab_size = 248320 + cfg.tie_word_embeddings = tie_emb + cfg.tensor_model_parallel_size = 4 + cfg.world_size = 8 + cfg.seq_length = 4096 + cfg.micro_batch = 2 + cfg.enable_sequence_parallel = True + cfg.computation_enable = False + cfg.add_bias_linear = False + cfg.moe_enable = False + cfg.moe_intermediate_size = 0 + cfg.moe_router_topk = 0 + cfg.num_experts = 0 + cfg.expert_model_parallel_size = 1 + cfg.full_attention_interval = 4 + cfg.linear_key_head_dim = linear_key_dim + cfg.linear_value_head_dim = linear_value_dim + cfg.linear_num_key_heads = 16 + cfg.linear_num_value_heads = linear_value_dim // head_dim if False else linear_value_dim + return cfg + + +def _build_moe_cfg35(hidden_size, intermediate_size, num_layers, + num_heads, num_kv, head_dim, moe_ff, topk, num_experts, + shared_intermediate): + cfg = types.SimpleNamespace() + cfg.hidden_size = hidden_size + cfg.intermediate_size = intermediate_size + cfg.num_hidden_layers = num_layers + cfg.num_attention_heads = num_heads + cfg.num_key_value_heads = num_kv + cfg.head_dim = head_dim + cfg.vocab_size = 248320 + cfg.tie_word_embeddings = False + cfg.tensor_model_parallel_size = 4 + cfg.expert_model_parallel_size = 4 + cfg.world_size = 32 + cfg.seq_length = 4096 + cfg.micro_batch = 2 + cfg.enable_sequence_parallel = True + cfg.computation_enable = False + cfg.add_bias_linear = False + cfg.moe_enable = True + cfg.moe_intermediate_size = moe_ff + cfg.moe_router_topk = topk + cfg.num_experts = num_experts + cfg.full_attention_interval = 4 + cfg.linear_key_head_dim = 128 + cfg.linear_value_head_dim = 128 + cfg.linear_num_key_heads = 16 + cfg.linear_num_value_heads = 32 + cfg.shared_expert_intermediate_size = shared_intermediate + return cfg + + +# --------------------------------------------------------------------------- +# Dense model tests +# --------------------------------------------------------------------------- + +class TestQwen35Dense: + """Qwen3.5 dense: hybrid GatedDeltaNet + full attention (3:1 pattern).""" + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,tie,lkd,lvd", QWEN35_DENSE) + def test_layer_routing_3to1_pattern(self, name, h, ff, L, n_heads, n_kv, hd, tie, lkd, lvd): + """Every 4th layer is full_attention; all others are GatedDeltaNet.""" + from workload_generator.mocked_model.training.MockedQwen3_5 import ( + Qwen3_5Params, Qwen3_5Model, Qwen3_5FullAttention, Qwen3_5GatedDeltaNet, + ) + + cfg = _build_dense_cfg35(h, ff, L, n_heads, n_kv, hd, tie, lkd, lvd) + model = Qwen3_5Model(cfg) + + full_count = 0; gdn_count = 0 + for i, layer in enumerate(model.layers): + attn = layer.attention + if isinstance(attn, Qwen3_5FullAttention): + full_count += 1 + assert (i + 1) % 4 == 0, f"{name}: layer {i} is full, expected at index {(i+1)%4} of 4" + elif isinstance(attn, Qwen3_5GatedDeltaNet): + gdn_count += 1 + assert (i + 1) % 4 != 0, f"{name}: layer {i} is GDN, expected NOT at multiple of 4" + + expected_full = L // 4 + expected_gdn = L - expected_full + assert full_count == expected_full, f"{name}: {full_count} full layers, expected {expected_full}" + assert gdn_count == expected_gdn, f"{name}: {gdn_count} GDN layers, expected {expected_gdn}" + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,tie,lkd,lvd", QWEN35_DENSE) + def test_gated_delta_net_zero_communication(self, name, h, ff, L, n_heads, n_kv, hd, tie, lkd, lvd): + """GatedDeltaNet forward/backward return empty workloads (all local compute).""" + from workload_generator.mocked_model.training.MockedQwen3_5 import ( + Qwen3_5Params, Qwen3_5Model, + ) + from workload_generator.generate_megatron_workload import MegatronWorkload + + cfg = _build_dense_cfg35(h, ff, L, n_heads, n_kv, hd, tie, lkd, lvd) + model = Qwen3_5Model(cfg) + args = _make_args35(h, L) + wl = MegatronWorkload(args, model)() + + gdn_items = [w for w in wl.workload + if w.stage and "gated_deltanet" in w.stage] + assert len(gdn_items) == 0, ( + f"{name}: GatedDeltaNet should produce 0 comm items, got {len(gdn_items)}" + ) + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,tie,lkd,lvd", QWEN35_DENSE) + def test_head_dim_256(self, name, h, ff, L, n_heads, n_kv, hd, tie, lkd, lvd): + """All full-attention layers use head_dim=256.""" + from workload_generator.mocked_model.training.MockedQwen3_5 import ( + Qwen3_5Params, Qwen3_5Model, Qwen3_5FullAttention, + ) + + cfg = _build_dense_cfg35(h, ff, L, n_heads, n_kv, hd, tie, lkd, lvd) + model = Qwen3_5Model(cfg) + + for layer in model.layers: + if isinstance(layer.attention, Qwen3_5FullAttention): + assert layer.attention.head_dim == 256, ( + f"{name}: expected head_dim=256, got {layer.attention.head_dim}" + ) + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,tie,lkd,lvd", QWEN35_DENSE) + def test_has_allgather_reduce_scatter(self, name, h, ff, L, n_heads, n_kv, hd, tie, lkd, lvd): + """Full-attention layers produce ALLGATHER and REDUCESCATTER.""" + from workload_generator.mocked_model.training.MockedQwen3_5 import ( + Qwen3_5Params, Qwen3_5Model, + ) + from workload_generator.generate_megatron_workload import MegatronWorkload + + cfg = _build_dense_cfg35(h, ff, L, n_heads, n_kv, hd, tie, lkd, lvd) + model = Qwen3_5Model(cfg) + args = _make_args35(h, L) + wl = MegatronWorkload(args, model)() + + ag = sum(1 for w in wl.workload if str(w.comm_type) == "CommType.all_gather") + rs = sum(1 for w in wl.workload if str(w.comm_type) == "CommType.reduce_scatter") + assert ag > 0, f"{name}: no all_gather items" + assert rs > 0, f"{name}: no reduce_scatter items" + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,tie,lkd,lvd", QWEN35_DENSE) + def test_per_layer_comm_count_formula(self, name, h, ff, L, n_heads, n_kv, hd, tie, lkd, lvd): + """Exact per-layer formula: GDN=3AG+2RS, Full=6AG+4RS (3:1 ratio).""" + from workload_generator.mocked_model.training.MockedQwen3_5 import ( + Qwen3_5Params, Qwen3_5Model, + ) + from workload_generator.generate_megatron_workload import MegatronWorkload + + cfg = _build_dense_cfg35(h, ff, L, n_heads, n_kv, hd, tie, lkd, lvd) + model = Qwen3_5Model(cfg) + args = _make_args35(h, L) + wl = MegatronWorkload(args, model)() + + ag = sum(1 for w in wl.workload if str(w.comm_type) == "CommType.all_gather") + rs = sum(1 for w in wl.workload if str(w.comm_type) == "CommType.reduce_scatter") + + full_layers = L // 4 + # GatedDeltaNet: 3 AG + 2 RS (MLP only, GDN returns empty Workload) + # Full attention: 6 AG + 4 RS (attention + MLP) + # Overhead: lm_head(2AG+1RS) + init/step(3AG+1RS) + exp_ag = L * 3 + full_layers * 3 + 5 + exp_rs = L * 2 + full_layers * 2 + 2 + + assert ag == exp_ag, f"{name}: AG {ag} != expected {exp_ag}" + assert rs == exp_rs, f"{name}: RS {rs} != expected {exp_rs}" + + +# --------------------------------------------------------------------------- +# MoE model tests +# --------------------------------------------------------------------------- + +class TestQwen35MoE: + """Qwen3.5 MoE: shared experts + GatedDeltaNet hybrid.""" + + @pytest.fixture(autouse=True) + def _skip_rank_fill(self, monkeypatch): + """Patch WorkloadGenerator.__call__ to skip _fill_ranks (pre-existing crash).""" + from workload_generator.workload_generator import WorkloadGenerator + + def _call_without_fill(self): + args = self.args + from log_analyzer.log import Workload, LogItem + from utils.utils import CommType + self.workload = Workload() + self.init() + self.workload.append(LogItem(comm_type=CommType.epoch_end)) + for _ in range(args.epoch_num): + if args.pipeline_model_parallel > 1 and args.frame != "collective_test": + self.with_pipeline_forward_backward() + else: + for _ in range(args.num_microbatches): + self.forward() + self.backward() + self.step() + self.workload.append(LogItem(comm_type=CommType.epoch_end)) + return self.workload + + monkeypatch.setattr(WorkloadGenerator, "__call__", _call_without_fill) + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,moe_ff,topk,nexp,shared", QWEN35_MOE) + def test_all_layers_use_moe(self, name, h, ff, L, n_heads, n_kv, hd, moe_ff, topk, nexp, shared): + """All layers in MoE config should route to MoE FFN.""" + from workload_generator.mocked_model.training.MockedQwen3_5 import ( + Qwen3_5Params, Qwen3_5Model, + ) + + cfg = _build_moe_cfg35(h, ff, L, n_heads, n_kv, hd, moe_ff, topk, nexp, shared) + model = Qwen3_5Model(cfg) + + moe_count = 0 + for layer in model.layers: + if hasattr(layer, 'mlp') and 'MoE' in layer.mlp.__class__.__name__: + moe_count += 1 + assert moe_count == L, f"{name}: {moe_count}/{L} layers use MoE, expected all" + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,moe_ff,topk,nexp,shared", QWEN35_MOE) + def test_alltoall_present(self, name, h, ff, L, n_heads, n_kv, hd, moe_ff, topk, nexp, shared): + """MoE layers produce AllToAll communication.""" + from workload_generator.mocked_model.training.MockedQwen3_5 import ( + Qwen3_5Params, Qwen3_5Model, + ) + from workload_generator.generate_megatron_workload import MegatronWorkload + + cfg = _build_moe_cfg35(h, ff, L, n_heads, n_kv, hd, moe_ff, topk, nexp, shared) + model = Qwen3_5Model(cfg) + args = _make_args35(h, L, ep=4, dp=2, moe=True) + wl = MegatronWorkload(args, model)() + + a2a = sum(1 for w in wl.workload if str(w.comm_type) == "CommType.all_to_all") + assert a2a > 0, f"{name}: no all_to_all items" + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,moe_ff,topk,nexp,shared", QWEN35_MOE) + def test_shared_expert_present_in_model(self, name, h, ff, L, n_heads, n_kv, hd, moe_ff, topk, nexp, shared): + """Model config includes shared_expert_intermediate_size.""" + from workload_generator.mocked_model.training.MockedQwen3_5 import ( + Qwen3_5Params, Qwen3_5Model, + ) + + cfg = _build_moe_cfg35(h, ff, L, n_heads, n_kv, hd, moe_ff, topk, nexp, shared) + model = Qwen3_5Model(cfg) + assert shared > 0, f"{name}: shared_expert_intermediate_size should be > 0" + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,moe_ff,topk,nexp,shared", QWEN35_MOE) + def test_alltoall_forward_backward_symmetry(self, name, h, ff, L, n_heads, n_kv, hd, moe_ff, topk, nexp, shared): + """AllToAll dispatch/combine should be symmetric between forward and backward.""" + from workload_generator.mocked_model.training.MockedQwen3_5 import ( + Qwen3_5Params, Qwen3_5Model, + ) + from workload_generator.generate_megatron_workload import MegatronWorkload + + cfg = _build_moe_cfg35(h, ff, L, n_heads, n_kv, hd, moe_ff, topk, nexp, shared) + model = Qwen3_5Model(cfg) + args = _make_args35(h, L, ep=4, dp=2, moe=True) + wl = MegatronWorkload(args, model)() + + moe_items = [w for w in wl.workload if w.stage and "MoE" in w.stage] + fwd_a2a = sum(1 for w in moe_items if w.stage.startswith("forward") + and str(w.comm_type) == "CommType.all_to_all") + bwd_a2a = sum(1 for w in moe_items if w.stage.startswith("backward") + and str(w.comm_type) == "CommType.all_to_all") + assert fwd_a2a == bwd_a2a, f"{name}: A2A asymmetry fwd={fwd_a2a} bwd={bwd_a2a}" + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,moe_ff,topk,nexp,shared", QWEN35_MOE) + def test_moe_backward_not_empty(self, name, h, ff, L, n_heads, n_kv, hd, moe_ff, topk, nexp, shared): + """MoE backward should not be empty (regression test for backward fix).""" + from workload_generator.mocked_model.training.MockedQwen3_5 import ( + Qwen3_5Params, Qwen3_5Model, + ) + from workload_generator.generate_megatron_workload import MegatronWorkload + + cfg = _build_moe_cfg35(h, ff, L, n_heads, n_kv, hd, moe_ff, topk, nexp, shared) + model = Qwen3_5Model(cfg) + args = _make_args35(h, L, ep=4, dp=2, moe=True) + wl = MegatronWorkload(args, model)() + + bwd_moe = [w for w in wl.workload + if w.stage and w.stage.startswith("backward") and "MoE" in w.stage] + assert len(bwd_moe) > 0, f"{name}: MoE backward is empty" + + +# --------------------------------------------------------------------------- +# SimAI .txt format tests for Qwen3.5 dense +# --------------------------------------------------------------------------- + +class TestQwen35SimAIGenerator: + """Qwen3.5 -> SIMAI_workload -> .txt (HYBRID_TRANSFORMER format).""" + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,tie,lkd,lvd", QWEN35_DENSE) + def test_txt_hybrid_transformer_header(self, name, h, ff, L, n_heads, n_kv, hd, tie, lkd, lvd): + from workload_generator.mocked_model.training.MockedQwen3_5 import ( + Qwen3_5Params, Qwen3_5Model, + ) + from workload_generator.SimAI_training_workload_generator import SIMAI_workload + import tempfile, os + + cfg = _build_dense_cfg35(h, ff, L, n_heads, n_kv, hd, tie, lkd, lvd) + model = Qwen3_5Model(cfg) + args = _make_args35(h, L); args.frame = "Qwen3.5"; args.num_layers = L + work = SIMAI_workload(model, args, None) + work.workload_generate() + + with tempfile.NamedTemporaryFile(suffix=".txt", delete=False, prefix="t35_") as tmp: + txt_path = tmp.name + try: + work.dump_file(txt_path.replace(".txt", "")) + with open(txt_path) as f: + first = f.readline().strip() + assert "HYBRID_TRANSFORMER" in first, f"{name}: missing header" + finally: + os.unlink(txt_path) + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,tie,lkd,lvd", QWEN35_DENSE) + def test_txt_has_comm_types(self, name, h, ff, L, n_heads, n_kv, hd, tie, lkd, lvd): + from workload_generator.mocked_model.training.MockedQwen3_5 import ( + Qwen3_5Params, Qwen3_5Model, + ) + from workload_generator.SimAI_training_workload_generator import SIMAI_workload + import tempfile, os + + cfg = _build_dense_cfg35(h, ff, L, n_heads, n_kv, hd, tie, lkd, lvd) + model = Qwen3_5Model(cfg) + args = _make_args35(h, L); args.frame = "Qwen3.5"; args.num_layers = L + work = SIMAI_workload(model, args, None) + work.workload_generate() + + with tempfile.NamedTemporaryFile(suffix=".txt", delete=False, prefix="t35_") as tmp: + txt_path = tmp.name + try: + work.dump_file(txt_path.replace(".txt", "")) + with open(txt_path) as f: + lines = f.read().strip().split("\n") + comm_types = set() + for i in range(2, len(lines)): + parts = lines[i].split("\t") + if len(parts) >= 7: + comm_types.add(parts[3]); comm_types.add(parts[6]) + assert "ALLGATHER" in comm_types, f"{name}: missing ALLGATHER" + assert "REDUCESCATTER" in comm_types, f"{name}: missing REDUCESCATTER" + finally: + os.unlink(txt_path) + + @pytest.mark.parametrize("name,h,ff,L,n_heads,n_kv,hd,tie,lkd,lvd", QWEN35_DENSE) + def test_txt_produces_workload(self, name, h, ff, L, n_heads, n_kv, hd, tie, lkd, lvd): + from workload_generator.mocked_model.training.MockedQwen3_5 import ( + Qwen3_5Params, Qwen3_5Model, + ) + from workload_generator.SimAI_training_workload_generator import SIMAI_workload + import tempfile, os + + cfg = _build_dense_cfg35(h, ff, L, n_heads, n_kv, hd, tie, lkd, lvd) + model = Qwen3_5Model(cfg) + args = _make_args35(h, L); args.frame = "Qwen3.5"; args.num_layers = L + work = SIMAI_workload(model, args, None) + work.workload_generate() + assert len(work.workload) > 0, f"{name}: empty workload" diff --git a/aicb/tests/test_rank_mapper.py b/aicb/tests/test_rank_mapper.py new file mode 100644 index 000000000..e5fd8186f --- /dev/null +++ b/aicb/tests/test_rank_mapper.py @@ -0,0 +1,67 @@ +"""Unit tests for rank_mapper.py — CommGroup-to-RankGenerator bridge.""" +import pytest +from utils.utils import CommGroup, RankGenerator + +def _make_gen(tp=1, dp=1, pp=1, ep=1, cp=1, order='tp-cp-ep-dp-pp'): + return RankGenerator(tp=tp, ep=ep, dp=dp, pp=pp, cp=cp, order=order) + +class TestGetRankListForCommGroup: + def test_tp_dp_pp_basic(self): + from utils.rank_mapper import get_rank_list_for_comm_group + rg = _make_gen(tp=2, dp=4, pp=1) + assert get_rank_list_for_comm_group(rg, CommGroup.tp_group, ref_rank=0) == [0, 1] + assert get_rank_list_for_comm_group(rg, CommGroup.dp_group, ref_rank=0) == [0, 2, 4, 6] + assert get_rank_list_for_comm_group(rg, CommGroup.pp_group, ref_rank=0) == [0] + + def test_world_size_1(self): + from utils.rank_mapper import get_rank_list_for_comm_group + rg = _make_gen(tp=1, dp=1, pp=1) + for cg in [CommGroup.tp_group, CommGroup.dp_group, CommGroup.pp_group]: + assert get_rank_list_for_comm_group(rg, cg) == [0] + + def test_comm_group_all(self): + from utils.rank_mapper import get_rank_list_for_comm_group + rg = _make_gen(tp=2, dp=4, pp=1) + assert get_rank_list_for_comm_group(rg, CommGroup.all) == list(range(8)) + + def test_embedding_group(self): + from utils.rank_mapper import get_rank_list_for_comm_group + rg = _make_gen(tp=2, dp=4, pp=1) + assert get_rank_list_for_comm_group(rg, CommGroup.embedding_group, ref_rank=0) == [0, 1] + + def test_unknown_comm_group_raises(self): + from utils.rank_mapper import get_rank_list_for_comm_group + rg = _make_gen() + with pytest.raises(ValueError, match="Unknown CommGroup"): + get_rank_list_for_comm_group(rg, "nonexistent_group") + + def test_ref_rank_not_found_raises(self): + from utils.rank_mapper import get_rank_list_for_comm_group + rg = _make_gen(tp=2, dp=2, pp=1) + with pytest.raises(ValueError, match="not found in any"): + get_rank_list_for_comm_group(rg, CommGroup.tp_group, ref_rank=999) + + def test_comm_group_none(self): + from utils.rank_mapper import get_rank_list_for_comm_group + assert get_rank_list_for_comm_group(_make_gen(), None) == [] + + def test_ep_enabled(self): + from utils.rank_mapper import get_rank_list_for_comm_group + rg = _make_gen(tp=1, dp=4, ep=2, order='tp-cp-ep-dp-pp') + ranks = get_rank_list_for_comm_group(rg, CommGroup.ep_group, ref_rank=0) + assert ranks == [0, 1] + ranks = get_rank_list_for_comm_group(rg, CommGroup.ep_dp_group, ref_rank=0) + assert len(ranks) == 4 + ranks = get_rank_list_for_comm_group(rg, CommGroup.ep_tp_group, ref_rank=0) + assert len(ranks) == 2 + +class TestBuildRankMappingTable: + def test_basic_table(self): + from utils.rank_mapper import build_rank_mapping_table + rg = _make_gen(tp=2, dp=4, pp=1) + rows = build_rank_mapping_table(rg) + assert len(rows) >= 5 + groups_by_name = {r['group']: r for r in rows} + assert groups_by_name['all_nodes']['size'] == 8 + assert groups_by_name['tp_group']['size'] == 2 + assert groups_by_name['dp_group']['size'] == 4 diff --git a/aicb/tests/test_regression_bugs.py b/aicb/tests/test_regression_bugs.py new file mode 100644 index 000000000..792a8d3d9 --- /dev/null +++ b/aicb/tests/test_regression_bugs.py @@ -0,0 +1,325 @@ +"""Regression tests for bugs found and fixed in 2025-06-15 research session. + +Covers: + 1. MOEMLP.backward() missing workloads.extend() (MockedMegatron.py) + 2. DeepSeekMoE.moe_mlp_backward() same bug (MockedDeepSeek.py) + 3. GatedDeltaNet returns empty Workload (MockedQwen3_5.py) + 4. SwiGLU intermediate_size backward compat (MockedMegatron.py) + 5. ffn_hidden_size -> intermediate_size rename (utils.py) +""" + +import types +import pytest + +from utils.utils import CommType +from workload_generator.mocked_model.MockedModel import MockedModel + + +# =========================================================================== +# 1. MOEMLP.backward() workloads.extend() regression +# =========================================================================== + +class TestMoEMLPBackward: + """Verify MOEMLP.backward() produces non-empty workloads (was broken).""" + + def test_megatron_moe_backward_not_empty(self): + """MOEMLP.backward() must include permutation/unpermutation comms.""" + from workload_generator.mocked_model.training.MockedMegatron import MOEMLP + + moe = MOEMLP( + batch_size=2, hidden_size=4096, tp=4, + expert_model_parallel_size=4, ffn_hidden_size=1536, + seq_len=4096, topk=8, num_experts=128, id=0, + ) + bwd = moe.backward() + assert len(bwd.workload) > 0, ( + f"MOEMLP.backward() is empty! " + f"The workloads.extend() fix may have been reverted." + ) + + def test_megatron_moe_forward_backward_parity(self): + """MOEMLP fwd and bwd should have similar comm counts.""" + from workload_generator.mocked_model.training.MockedMegatron import MOEMLP + + moe = MOEMLP( + batch_size=2, hidden_size=4096, tp=4, + expert_model_parallel_size=4, ffn_hidden_size=1536, + seq_len=4096, topk=8, num_experts=128, id=0, + ) + fwd = moe.forward() + bwd = moe.backward() + # Forward has preprocess all_gather + permutation + unpermutation + # Backward has permutation + unpermutation (same ops, reverse) + # Ratio should be close to 1.0 + assert len(bwd.workload) > 0 + assert abs(len(bwd.workload) - len(fwd.workload)) <= 1, ( + f"MOEMLP fwd/bwd asymmetry: fwd={len(fwd.workload)} bwd={len(bwd.workload)}" + ) + + def test_megatron_moe_alltoall_symmetry(self): + """A2A ops in MoE forward and backward should be equal.""" + from workload_generator.mocked_model.training.MockedMegatron import MOEMLP + + moe = MOEMLP( + batch_size=2, hidden_size=4096, tp=4, + expert_model_parallel_size=4, ffn_hidden_size=1536, + seq_len=4096, topk=8, num_experts=128, id=0, + ) + fwd = moe.forward() + bwd = moe.backward() + + fwd_a2a = sum(1 for w in fwd.workload + if str(w.comm_type) == "CommType.all_to_all") + bwd_a2a = sum(1 for w in bwd.workload + if str(w.comm_type) == "CommType.all_to_all") + assert fwd_a2a == bwd_a2a, ( + f"A2A asymmetry in MOEMLP: fwd={fwd_a2a} bwd={bwd_a2a}" + ) + + +# =========================================================================== +# 2. DeepSeekMoE backward regression +# =========================================================================== + +class TestDeepSeekMoEBackward: + """Verify DeepSeekMoE.moe_mlp_backward() produces non-empty workloads.""" + + def test_deepseek_moe_backward_not_empty(self): + """DeepSeekMoE.moe_mlp_backward() must include perm/unperm comms.""" + from workload_generator.mocked_model.training.MockedDeepSeek import DeepSeekMoE + + moe = DeepSeekMoE( + hidden_size=7168, ffn_hidden_size=18432, tp=8, + expert_model_parallel_size=32, seq_len=4096, + batch_size=2, topk=8, num_experts=256, id=0, + n_shared_expert=0, sequence_parallel_enabled=True, + computation_enable=False, add_bias_linear=False, + ) + fwd = moe.forward() + bwd = moe.backward() + assert len(bwd.workload) > 0, ( + f"DeepSeekMoE.backward() is empty! " + f"The workloads.extend() fix may have been reverted." + ) + assert len(fwd.workload) > 0 + + +# =========================================================================== +# 3. GatedDeltaNet empty Workload +# =========================================================================== + +class TestGatedDeltaNetComms: + """Verify GatedDeltaNet layers produce zero communication.""" + + def test_gated_delta_net_forward_empty(self): + """Qwen3.5 GatedDeltaNet forward() must return empty Workload.""" + from workload_generator.mocked_model.training.MockedQwen3_5 import ( + Qwen3_5GatedDeltaNet, + ) + + gdn = Qwen3_5GatedDeltaNet( + hidden_size=4096, + linear_key_head_dim=128, + linear_value_head_dim=128, + linear_num_key_heads=16, + linear_num_value_heads=32, + linear_conv_kernel_dim=4, + layer_id=1, + ) + fwd = gdn.forward() + assert len(fwd.workload) == 0, ( + f"GatedDeltaNet forward should return empty Workload, got {len(fwd.workload)}" + ) + + def test_qwen35_dense_comm_count(self): + """Qwen3.5-9B: 82 fwd = 1 + 8*4 + 24*2 + 1.""" + from workload_generator.mocked_model.training.MockedQwen3_5 import ( + Qwen3_5Params, Qwen3_5Model, + ) + + cfg = Qwen3_5Params() + cfg.hidden_size = 4096 + cfg.intermediate_size = 12288 + cfg.num_hidden_layers = 32 + cfg.num_attention_heads = 16 + cfg.num_key_value_heads = 4 + cfg.head_dim = 256 + cfg.vocab_size = 248320 + cfg.full_attention_interval = 4 + cfg.linear_key_head_dim = 128 + cfg.linear_value_head_dim = 128 + cfg.linear_num_key_heads = 16 + cfg.linear_num_value_heads = 32 + cfg.linear_conv_kernel_dim = 4 + cfg.tensor_model_parallel_size = 8 + cfg.world_size = 8 + cfg.seq_length = 4096 + cfg.micro_batch = 2 + cfg.enable_sequence_parallel = True + cfg.computation_enable = False + cfg.add_bias_linear = False + cfg.moe_enable = False + + model = Qwen3_5Model(cfg) + fwd = model.forward() + assert len(fwd.workload) == 82, ( + f"Qwen3.5-9B fwd={len(fwd.workload)}, expected 82" + ) + + +# =========================================================================== +# 4. SwiGLU intermediate_size backward compat +# =========================================================================== + +class TestSwiGLUBackwardCompat: + """Verify intermediate_size rename preserves backward compat.""" + + def test_old_ffn_hidden_size_still_works(self): + """Old config.ffn_hidden_size should work via getattr fallback.""" + from workload_generator.mocked_model.training.MockedMegatron import MegatronModel + + class OldConfig: + hidden_size = 4096 + ffn_hidden_size = 4 * 4096 # old API + num_layers = 32 + num_attention_heads = 32 + padded_vocab_size = 128256 + tensor_model_parallel_size = 8 + seq_length = 4096 + micro_batch = 2 + enable_sequence_parallel = True + computation_enable = False + add_bias_linear = False + moe_enable = False + expert_model_parallel_size = 1 + moe_router_topk = 0 + num_experts = 0 + moe_grouped_gemm = True + moe_intermediate_size = None + # no swiglu attr, no intermediate_size attr + + model = MegatronModel(OldConfig()) + fwd = model.forward() + assert len(fwd.workload) == 129, ( + f"Old API fwd={len(fwd.workload)}, expected 129" + ) + + def test_new_intermediate_size_produces_same_result(self): + """New intermediate_size with same value = old ffn_hidden_size.""" + from workload_generator.mocked_model.training.MockedMegatron import MegatronModel + + class NewConfig: + hidden_size = 4096 + intermediate_size = 4 * 4096 # new API + num_layers = 32 + num_attention_heads = 32 + padded_vocab_size = 128256 + tensor_model_parallel_size = 8 + seq_length = 4096 + micro_batch = 2 + enable_sequence_parallel = True + computation_enable = False + add_bias_linear = False + moe_enable = False + expert_model_parallel_size = 1 + moe_router_topk = 0 + num_experts = 0 + moe_grouped_gemm = True + swiglu = False + moe_intermediate_size = None + + model = MegatronModel(NewConfig()) + fwd = model.forward() + + # Same as old API + class OldConfig: + hidden_size = 4096 + ffn_hidden_size = 4 * 4096 + num_layers = 32 + num_attention_heads = 32 + padded_vocab_size = 128256 + tensor_model_parallel_size = 8 + seq_length = 4096 + micro_batch = 2 + enable_sequence_parallel = True + computation_enable = False + add_bias_linear = False + moe_enable = False + expert_model_parallel_size = 1 + moe_router_topk = 0 + num_experts = 0 + moe_grouped_gemm = True + moe_intermediate_size = None + + old_model = MegatronModel(OldConfig()) + old_fwd = old_model.forward() + + assert len(fwd.workload) == len(old_fwd.workload), ( + f"New API fwd={len(fwd.workload)} != old API fwd={len(old_fwd.workload)}" + ) + + def test_swiglu_sizing(self): + """SwiGLU with intermediate_size=12288 correctly uses 2*intermediate for column.""" + from workload_generator.mocked_model.training.MockedMegatron import ( + MegatronMlp, MegatronColumnLinear, MegatronRowLinear, + ) + + mlp = MegatronMlp( + hidden_size=4096, intermediate_size=12288, + tp=8, seq_len=4096, batch_size=2, layer_id=0, + sequence_parallel_enabled=True, computation_enable=False, + add_bias_linear=False, swiglu=True, + ) + assert mlp.dense_h_to_4h.output_size == 2 * 12288, ( + f"SwiGLU column output should be 2*intermediate, got {mlp.dense_h_to_4h.output_size}" + ) + assert mlp.dense_4h_to_h.input_size == 12288, ( + f"SwiGLU row input should be intermediate, got {mlp.dense_4h_to_h.input_size}" + ) + + +# =========================================================================== +# 5. CLI alias regression +# =========================================================================== + +class TestCLIAliases: + """Verify num_hidden_layers and padded_vocab_size aliases work.""" + + def test_num_hidden_layers_alias(self): + """CLI --num_layers should alias to num_hidden_layers for Qwen3.""" + import types + + # Simulate an argparse namespace with only CLI args (no config file) + args = types.SimpleNamespace() + args.num_layers = 36 + args.hidden_size = 4096 + args.vocab_size = 151936 + args.num_attention_heads = 32 + args.swiglu = True + args.intermediate_size = None + args.ffn_hidden_size = None + args.padded_vocab_size = None + args.tensor_model_parallel_size = 8 + args.make_vocab_size_divisible_by = 128 + + # Apply the resolution logic from get_params() + if args.num_attention_heads is None: + args.num_attention_heads = args.num_layers + args.num_hidden_layers = args.num_layers + if not hasattr(args, 'padded_vocab_size') or args.padded_vocab_size is None: + # get_padded_vocab_size needs make_vocab_size_divisible_by + args.make_vocab_size_divisible_by = 128 + import utils.utils + args.padded_vocab_size = utils.utils.get_padded_vocab_size(args) + if args.intermediate_size is None: + args.intermediate_size = args.ffn_hidden_size + if args.intermediate_size is None: + if args.swiglu: + args.intermediate_size = int((4 * args.hidden_size * 2 / 3) / 64) * 64 + else: + args.intermediate_size = 4 * args.hidden_size + + assert hasattr(args, 'num_hidden_layers'), "num_hidden_layers alias missing" + assert args.num_hidden_layers == 36, f"num_hidden_layers={args.num_hidden_layers}" + assert args.padded_vocab_size is not None, "padded_vocab_size not set" + assert args.intermediate_size > 0, "intermediate_size should be computed" diff --git a/aicb/training/tutorial.md b/aicb/training/tutorial.md new file mode 100755 index 000000000..1ea532e91 --- /dev/null +++ b/aicb/training/tutorial.md @@ -0,0 +1,332 @@ +# Introduction +AI Communication Benchmark is a specialized communication benchmarking suite designed for artificial intelligence (AI) scenarios, primarily used to evaluate the performance of communication stacks. This suite not only provides detailed performance metrics but also assists developers in quickly identifying and diagnosing potential performance bottlenecks and issues within the communication stack. By simulating real-world communication traffic patterns during AI training and inference processes, this benchmark accurately reflects the communication stack's actual performance under conditions of high concurrency and large data transfer volumes. Whether it's inter-node communication in distributed computing or data synchronization in large-scale model training, this benchmarking suite offers effective performance evaluation and optimization recommendations to help users enhance overall system efficiency and stability. + +# Environment Setup +Before setting up the environment, first pull the code repository to your local machine and then proceed with the environment configuration: +``` +git clone https://github.com/aliyun/aicb.git +``` +For the environment, if you are only generating workloads, no additional dependencies are needed. However, other functionalities require dependencies such as PyTorch, CUDA, NCCL, and NVIDIA APEX. Therefore, you can set up an appropriate runtime environment either by configuring your local environment or using Docker. +## Setting the environment using a Dockerfile. +``` +docker build -t aicb:v0.0.1 . +docker run --gpus all --net host --shm-size 16g -it --rm aicb:v0.0.1 +``` +## Setting the Environment Locally +For a local environment, you will need Python >= 3.8, CUDA Version >= 11.8, PyTorch >= 2.0.0, and NVIDIA APEX. +## Using Official Docker +You can also create the required Docker environment using [NGC's PyTorch container ](https://ngc.nvidia.com/catalog/containers/nvidia:pytorch), where pytorch:xx.xx should be >= pytorch:23.08. +``` +docker pull nvcr.io/nvidia/pytorch:xx.xx-py3 +docker run --gpus all -it --rm -v /path/to/AICBench:/workspace/AICBench nvcr.io/nvidia/pytorch:xx.xx-py3 +``` + +# Basic Usage +## Physical Execution +When running on a physical machine, additional configuration for PyTorch-related environment variables is required. This can be done by explicitly specifying them in a script or by adding the environment variables directly. The following table lists the required environment variables: + +| Parameter Name | Description | +|-----------------|-------------------------------| +| nnodes | Number of nodes | +| node_rank | Rank number of the node | +| nproc_per_node | Number of GPUs per node | +| master_addr | Address of the master node | + + +### Quick start for single-node execution +The script for running AICB on a physical machine is：[/scripts/megatron_workload_with_aiob.sh](../scripts/megatron_workload_with_aiob.sh) + +We provide four pre-existing models (7/13/22/175/)B, moe, and DeepSeek (16/236/671)B to quickly launch and run on a physical machine, which can be specified using the parameter `--model_size`. Additionally, the Megatron parallel framework supports enabling the aiob_enable option to obtain the computation time for each operation of the actual model. Without using aiob, only fixed waiting times can be filled. Alternatively, when AIOB is enabled, you can specify `--comp_filepath` to fill in the corresponding computation time. +Below is an example of generating a Workload with a model size of 13B, tp 8, pp 1, a total GPU count of 8, gbs 2, mbs 1, sequence length of 4096, with flash_attn and swiglu enabled, and using AIOB to obtain the computation time for each operation of the actual model. +``` bash +export MASTER_ADDR=127.0.0.1 +export MASTER_PORT=23089 +export WORLD_SIZE=1 +export RANK=0 + +sh ./scripts/megatron_gpt.sh \ +-m 13 --world_size 8 --tensor_model_parallel_size 8 --pipeline_model_parallel 1 \ +--frame Megatron --global_batch 2 \ +--micro_batch 1 --seq_length 4096 \ +--swiglu --use_flash_attn --aiob_enable +``` +### Quick start for multi-node execution +The script used for multi-node execution is：[run_in_cluster.py](../scripts/run_in_cluster.py) + +Steps： +1. First, install the batch distribution commands (such as pssh and pscp). +2. Edit the `iplist` file of the cluster to be used, adding an accessible IP address of each machine per line to the iplist. +3. Modify [run_in_cluster.py](../scripts/run_in_cluster.py) to specify the image name and paths to `iplist` file and AICB home directory. Please refer to the doc of [run_in_cluster.py](../scripts/run_in_cluster.py) for more details. +4. Modify [run_suites.py](../run_suites.py) to select the workload to run (default: no workload). +5. Copy the `iplist` and AICB source code to each machine (e.g., using pscp). +4. Run the command just like this: `pssh -i -h /path/to/iplist -o out -e err -t 0 "cd /path/to/aicb && python run_in_cluster.py"`. Remember to replace `/path/to/iplist` and `/path/to/aicb` with the actual path on your machine. + +The specific command to be run on each machine can be modified in the highlighted section: + +![Scaling Graph](../images/tutorial_7.png) +### Logs and Results +After each communication is completed, the program will print the relevant logs for this communication. The output format is as follows and contains almost all the information about the communication operation: +* communication type +* communication group +* Message size +* Communication execution time +* [Throughput](https://github.com/NVIDIA/nccl-tests/blob/master/doc/PERFORMANCE.md) + +![Scaling Graph](../images/tutorial_1.png) + +After all communications are completed, information about the use case will be summarized. +* This includes the overall runtime and an analysis of the time for each iteration, allowing for a clear view of whether each iteration runs normally and if there are any jitters. +![Scaling Graph](../images/tutorial_2.png) +* Time for each communication type +First, distinguish between the model training phases: the init phase and the train phase. Then, summarize the collective communications performed in each phase, including their corresponding message sizes, frequencies, and specific average latencies, maximum and minimum values, etc. This helps to pinpoint which type of collective communication operation in which message segment is causing anomalies, facilitating further investigation and troubleshooting. +![Scaling Graph](../images/tutorial_3.png) +#### File outputs +The file outputs include two different types of files: .csv file. +1. The CSV files are saved in: +`results/comm_logs/megatron_gpt_13B_8n_log.csv`,And you can also see the execution time, the execution phase, as well as the algorithmic bandwidth (algbw) and bus bandwidth (busbw) belonging to different comm_group and different comm_type.It also includes the computation time for each part of the model and the computation phase it belongs to. + +![Scaling Graph](../images/tutorial_4.png) + +Inaddition to the aforementioned details, a .csv file is provided for detailed analysis of the results. Here’s how to work with it: + 1. Reading _workload.csv Log: + * You can read the _workload.csv log file by invoking log_analyzer.log.Workload.load(filename). + * This will return Workload and args. + * args contains the parameters used for training input. + * Workload consists of the generated intermediate results. + 2. Reading _log.csv Log: + * You can read the _log.csv log file by invoking log_analyzer.log.Log.load(filename). + * This will return a Log object, containing: + * comm_logs: List[LogItem]: This is a list of all generated logs. + * epoch_times: List[int]: This lists the time taken for each iteration. The first iteration typically represents initialization, which might show different communication behavior compared to subsequent iterations, potentially leading to differences in time. + * comm_log_each_epoch: List[List[LogItem]]: This is a list where each item corresponds to the communication logs for each iteration. If one iteration has a significantly different time compared to others, you can analyze this specific iteration to identify the communication causing the discrepancy. +By leveraging these log files and parsing methods, you can perform a thorough and detailed analysis of the training process, identifying any abnormalities or areas for optimization. +## Generate Workload for Simulation(SimAI) +### Quick start +AICB's script for generating Workload is: `./scripts/megatron_workload_with_aiob.sh` +We provide four pre-existing models (7/13/22/175)B and DeepSeek (16/236/671)B to quickly generate the corresponding Workload, which can be specified using the parameter `--model_size`.The computation part of the model can be selected to use aiob via--aiob_enableWhen not using aiob, the default fixed time is used to fill the Workload.When `--aiob_enable` is enabled, if `--comp_filepath` is not specified, the current GPU's computation time will be used to fill the Workload +Below is an example of generating a Workload with a model size of 7B, tp 4, pp 1, a total GPU count of 4096, gbs 8192, mbs 1, sequence length of 4096, with flash_attn, swiglu, and aiob enabled, and reading Example.txt as the computation time. +```bash +sh ./scripts/megatron_workload_with_aiob.sh -m 7 \ +--world_size 4096 --tensor_model_parallel_size 4 --pipeline_model_parallel 1 \ +--frame Megatron --global_batch 8192 \ +--micro_batch 1 --seq_length 4096 --swiglu \ +--use_flash_attn --aiob_enable \ +--comp_filepath workload/aiob_inputs/Example.txt +``` +### Workload +The generated Workload result is saved in:`results/workload/gpt_7B-world_size4096-tp4-pp1-gbs8192-mbs1-seq4096-flash_attn-True.txt` +![Scaling Graph](../images/tutorial_6.png) + +## Run AICB with customized cases +In addition to the quick start options, you can also customize the model parameters in detail to run on physical machines or generate the required workloads for simulation and analysis. This flexibility allows you to tailor the workloads specifically to your needs, whether you are experimenting with different configurations of large language models, testing various parallel frameworks, or optimizing your runtime environment. Customizing parameters provides deeper insights and greater control over the benchmarking and simulation processes, enabling more precise performance tuning and analysis. + +### Parameters +The main parameters for AICB are as follows: + +| Category | Parameter Name | Description | +|------------------------------|-----------------------------------|-----------------------------------------------------------------------------| +| Name | frame | DeepSpeed/Megatron/DeepSeek | +| | model_name | Llama/GPT/... | +|Training Parameters | world_size | Total number of GPUs | +| | global_batch | Total batch size for training | +| | micro_batch | Batch size per model instance (local batch size). | +| | epoch_num | Number of iterations | +| Model parameters | model_size | Model size (7/13/65/175/270)B and moe | +| | num_layers | Number of transformer layers. | +| | hidden_size | Transformer hidden size. | +| | num_attention_heads | Number of transformer attention heads. | +| | seq_length | Maximum sequence length to process. | +| | vocab_size | Size of vocab before EOD or padding. | +| | max_position_embeddings | Maximum number of position embeddings to use. | +| | ffn_hidden_size | Transformer Feed-Forward Network hidden size. | +| Megatron parallel parameters | tensor_model_parallel_size | Degree of tensor model parallelism. | +| | pipeline_model_parallel | Degree of pipeline model parallelism. | +| | enable_sequence_parallel | Enable sequence parallel optimization. | +| Megatron optimization parameters | use_flash_attn | Use FlashAttention implementation of attention. | +| | swiglu | Use gated linear units and SiLU activation instead of default gelu | +| | openai_gelu | Use OpenAI's GeLU implementation. | +| | onnx_safe | Use workarounds for known problems with Torch ONNX exporter | +| | squared_relu | Use squared relu activation instead of default gelu | +| | bias_gelu_fusion | Enable bias and gelu fusion. | +| | gated_linear_unit | Enable when use swiglu | +| MoE | expert_model_parallel_size | Degree of expert model parallelism | +| | moe_enable | Enable MoE | +| | num_experts | Number of Experts in MoE (None means no MoE) | +| | moe_router_topk | Number of experts to route to for each token. | +| | moe_grouped_gemm | When there are multiple experts per rank, compress multiple local (potentially small) gemms in a single kernel| +| DeepSpeed parameters | zero_stage,reduce_bucket_size | choose zero optimizer stage | +| | allgather_bucket_size | Optimizes communication efficiency and memory usage during all-gather operations For stage 1/2 only | +| | prefetch_bucket_size, param_persistence_threshold, model_persistence_threshold, max_live_parameters | For stage 3 only. Control the number of prefetch parameters. Control the size of all_gather and reduce_scatter | +| DeepSeek parameters | qk_rope_dim | RoPE dimention for QK | +| | qk_nope_dim | non-RoPE (aka NoPE) dimention for QK | +| | q_lora_rank | Q down projection (aka compression, LoRA) dimention for Multi-Head Latent Attention (MLA) | +| | kv_lora_rank | KV down projection (aka compression, LoRA) dimention for MLA | +| | v_head_dim | V head dimention | +| | n_shared_expert | Number of shared experts per MoE layer | +| | n_dense_layer | Number of transformer blocks with dense MLP instead of MoE | +| Other | aiob_enable | Enable AIOB to obtain computation time | +| | comp_filepath | Use aiob_lib to get operation compute time | + +### Running on physical GPU clusters +The current entry file for running custom cases is [aicb.py](../aicb.py). By using this file, you can flexibly choose more parameters for tuning. +```bash +# DeepSpeed Stage 2 Example +torchrun \ +--nnodes ${WORLD_SIZE} \ +--node_rank ${RANK} \ +--nproc_per_node gpu \ +--master_addr ${MASTER_ADDR} \ +--master_port ${MASTER_PORT} \ +./aicb.py --frame=DeepSpeed --stage=$stage \ +--world_size=$((NNODES*8)) --global_batch=$global_batch --epoch_num=$epoch_num \ +--num_layers=$num_layers --hidden_size=$hidden_size \ +--ffn_hidden_size=$ffn_hidden_size --num_attention_heads=$attention_heads \ +--reduce_bucket_size=$bucket_size --allgather_bucket_size=$bucket_size + +# Megatron Example +torchrun \ +--nnodes $WORLD_SIZE \ +--node_rank $RANK \ +--nproc_per_node gpu \ +--master_addr $MASTER_ADDR \ +--master_port $MASTER_PORT \ +./aicb.py --frame=Megatron --world_size=$((WORLD_SIZE*8)) --tensor_model_parallel_size=$tensor_model_parallel_size \ + --micro_batch=$batch_size --global_batch=$((WORLD_SIZE*8*batch_size/tensor_model_parallel_size)) --epoch_num=$epoch_num --swiglu \ + --num_layers=$num_layers --hidden_size=$hidden_size --ffn_hidden_size=$ffn_hidden_size --num_attention_heads=$num_attention_heads \ + $sp_enable --seq_len=$seq_len --vocab_size=$vocab_size --aiob_enable=$enable + +# MoE Example +torchrun \ +--nnodes $WORLD_SIZE \ +--node_rank $RANK \ +--nproc_per_node gpu \ +--master_addr $MASTER_ADDR \ +--master_port $MASTER_PORT \ +./aicb.py --frame=Megatron --world_size=$((WORLD_SIZE*8)) --tensor_model_parallel_size=$tensor_model_parallel_size --expert_model_parallel_size=$expert_model_parallel_size \ +--moe_enable=$moe_enable --num_experts=$num_experts --moe_router_topk=$moe_router_topk --moe_grouped_gemm=$moe_grouped_gemm \ + --micro_batch=$batch_size --global_batch=$((WORLD_SIZE*8*batch_size/tensor_model_parallel_size)) --epoch_num=$epoch_num --swiglu \ + --num_layers=$num_layers --hidden_size=$hidden_size --ffn_hidden_size=$ffn_hidden_size --num_attention_heads=$num_attention_heads \ + $sp_enable --seq_len=$seq_len --vocab_size=$vocab_size --aiob_enable=$enable + +# DeepSeek Example +torchrun \ +--nnodes $WORLD_SIZE \ +--node_rank $RANK \ +--nproc_per_node gpu \ +--master_addr $MASTER_ADDR \ +--master_port $MASTER_PORT \ +./aicb.py \ + --frame=DeepSeek --world_size=$((WORLD_SIZE*8)) --tensor_model_parallel_size=$tensor_model_parallel_size --expert_model_parallel_size=$expert_model_parallel_size \ + --moe_enable=$moe_enable --num_experts=$num_experts --moe_router_topk=$moe_router_topk \ + --micro_batch=$batch_size --global_batch=$((WORLD_SIZE*8*batch_size/tensor_model_parallel_size)) \ --epoch_num=$epoch_num --swiglu \ + --num_layers=$num_layers --hidden_size=$hidden_size --ffn_hidden_size=$ffn_hidden_size --num_attention_heads=$num_attention_heads \ + $sp_enable --seq_len=$seq_len --vocab_size=$vocab_size --aiob_enable=$enable +``` +### Generating Workloads +In AICB, the generation of Workloads is divided into two types: one is to generate Workloads that can be used for simulation and analysis, and the other is to generate general-purpose Workloads that contain detailed information about various communication and computation aspects. +By providing the detailed parameters of the model, you can generate a general-purpose Workload that includes detailed information about various communications and computations. The general-purpose workload allows you to analyze the model's computational and communication performance under different parameters, and customize the workload files for run-time tuning and optimization. This can be achieved by using the following files: +[generate_deepspeed_stage3_workload](../workload_generator/generate_deepspeed_stage3_workload.py),[generate_deepspeed_stage1_2_workload](../workload_generator/generate_deepspeed_stage1_2_workload.py),[generate_megatron_workload](../workload_generator/generate_megatron_workload.py) + +Here is an example: +```bash +python -m workload_generator.SimAI_training_workload_generator \ + --model_name GPT-13B --frame=Megatron \ + --world_size=16 --tensor_model_parallel_size=2 --pipeline_model_parallel=1 --global_batch=16 \ + --micro_batch=1 --num_layers=40 --seq_length=2048 \ + --hidden_size=5120 --epoch_num=1 \ + --use-distributed-optimizer --num_attention_heads=40 \ + --aiob_enable --use_flash_attn --swiglu +``` +#### Workload Files +The generated Workload files are saved in the `results/mocked_workload` directory. +Here is an explanation of a the generated Workload file: +![Scaling Graph](../images/tutorial_5.png) + +### Creating customized Models +AICB offers remarkable extensibility. In addition to supporting GPT and LLaMA series models, it also allows for the creation of workloads for custom model architectures. This flexibility means you can adapt AICB to generate and test communication and computation patterns for a wide variety of models, beyond the pre-configured options, making it an invaluable tool for benchmarking and optimizing diverse AI training frameworks. +Custom models can be built using the MockedParam and MockedModel base classes. For specific implementation details, you can refer to the existing MockedMegatron and MockedDeepSpeed implementations. + +Here is an example of a DeepSpeed Llama model: +``` +DeepspeedForCausalLM + | + v ++----------------------+ +| Linear: embed_tokens | ++----------------------+ + | + v ++--------------------------------------+ +| DeepspeedModel | +| +----------------------------------+ | +| | Linear: embed_tokens | | +| +----------------------------------+ | +| | DeepspeedDecoderLayer x N | | +| +----------------------------------+ | +| | Linear: norm | | ++--------------------------------------+ + | + v ++--------------------------------------------------+ +| DeepspeedDecoderLayer | +| +----------------------------------------------+ | +| | Linear: input_layernorm | | +| +----------------------------------------------+ | +| | DeepspeedAttention | | +| | +----------------------------------------+ | | +| | | Linear: q_proj | | | +| | | Linear: k_proj | | | +| | | Linear: v_proj | | | +| | | Linear: o_proj | | | +| | +----------------------------------------+ | | +| +----------------------------------------------+ | +| | Linear: post_attn_norm | | +| +----------------------------------------------+ | +| | DeepspeedMLP | | +| | +----------------------------------------+ | | +| | | Linear: gate_proj | | | +| | | Linear: down_proj | | | +| | | Linear: up_proj | | | +| | +----------------------------------------+ | | ++--------------------------------------------------+ + | + v ++----------------------+ +| Linear: lm_head | ++----------------------+ + +``` + +Besides Model, you alse need to mocked the entire training process. The training process for all frameworks is abstracted into the following steps: `init, forward, backward, and step` (excluding pipeline parallelism). We need to further elaborate on the communications that occur in each step with workload items. +In the code, each workload primarily consists of three components: +1. Communication Information: + This primarily includes information related to collective communication activities such as `comm_type, comm_group, comm_group_size, and msg_size`. +2. Additional Information: + This includes supplementary information, such as the source node for broadcast communications and the time for compute operations. +3. Runtime Information: + This mainly consists of runtime-specific details like `elapsed_time, algo_bw, and bus_bw` which indicate the actual performance and state of the collective communication activities. +Here is a brief example of training process and workload item: +```python +trainer.init() +for _ in range(epoch_num): + if pipeline_model_parallel > 1: + trainer.with_pipeline_forward_backward() + else: + for _ in range(num_microbatches): + trainer.forward() + trainer.backward() + trainer.step() +workload.append({ + "operation": "init", # Corresponds to one of init, forward, backward, step + "comm_type": CommType.broadcast, # The type of communication + "call_func": "_broadcast_model", # The function invoked in the source code + "msg_size": param.msg_size(), # The size of the communication message + "comm_group": CommGroup.dp_group,# The communication group + "src": 0, # Optional: Only for broadcast, specifies the source node + "additional": send_next # Optional: Specifies the corresponding operation in pipeline parallelism +}) +``` + +# Troubleshooting & FAQs +N/A + diff --git a/aicb/tuning/__init__.py b/aicb/tuning/__init__.py new file mode 100644 index 000000000..bebf983d6 --- /dev/null +++ b/aicb/tuning/__init__.py @@ -0,0 +1,18 @@ +"""AICB Tunability Package (F003). + +Decorator-pattern wrappers for injecting performance variability +into generated workloads without modifying core mocked_model logic. + +Components: +- TunabilityWrapper: orchestrates straggler + scaling + variability +- WorkloadScaler: post-generation DP/TP dimension reconfiguration +- VariabilityInjector: Monte Carlo noise on operation durations + +Reference: .omc/research/aicb-model-extensibility.md, Section 4.4 (F003). +""" + +from .wrapper import TunabilityWrapper +from .scaler import WorkloadScaler +from .variability import VariabilityInjector + +__all__ = ["TunabilityWrapper", "WorkloadScaler", "VariabilityInjector"] diff --git a/aicb/tuning/scaler.py b/aicb/tuning/scaler.py new file mode 100644 index 000000000..48f2a0d40 --- /dev/null +++ b/aicb/tuning/scaler.py @@ -0,0 +1,142 @@ +"""F003: Workload Scaler. + +Post-generation parallelism reconfiguration. Changes DP/TP/PP +dimensions without regenerating layer structure, preserving total +FLOP count while recalculating communication sizes and participant +groups. + +Key invariant: total_gpus = new_dp * new_tp * new_pp +""" + +import math +from typing import List, Optional +from log_analyzer.log import LogItem, CommType, CommGroup + + +class WorkloadScaler: + """Reconfigures parallelism dimensions on an existing workload. + + Recalculates communication topology (comm_group, comm_group_size, + msg_size) for changed DP/TP/PP dimensions. Preserves compute ops + unchanged (same FLOPs per layer regardless of parallelism). + """ + + def __init__(self, log_items: List[LogItem]): + self._items = log_items + self._original_config: Optional[dict] = None + + def scale( + self, + new_dp: Optional[int] = None, + new_tp: Optional[int] = None, + new_pp: Optional[int] = None, + total_gpus: Optional[int] = None, + ) -> List[LogItem]: + """Apply parallelism scaling. + + At least one of new_dp, new_tp, new_pp must be provided. + Missing dimensions are derived from total_gpus constraint. + + Args: + new_dp: New data-parallel size. + new_tp: New tensor-parallel size. + new_pp: New pipeline-parallel size. + total_gpus: Total GPU count override. + + Returns: + The modified LogItem list. + """ + # Detect original parallelism dimensions from comm groups + orig = self._detect_original_parallelism() + if orig is None: + return self._items # Cannot detect, skip scaling + + # Fill missing dimensions + dp = new_dp if new_dp is not None else orig["dp"] + tp = new_tp if new_tp is not None else orig["tp"] + pp = new_pp if new_pp is not None else orig["pp"] + gpus = total_gpus if total_gpus is not None else dp * tp * pp + + if gpus != dp * tp * pp: + raise ValueError( + f"total_gpus ({gpus}) != dp*tp*pp ({dp}*{tp}*{pp} = {dp*tp*pp})" + ) + + if dp == orig["dp"] and tp == orig["tp"] and pp == orig["pp"]: + return self._items # No change needed + + # Recalculate communication topology for each item + for item in self._items: + self._rescale_item(item, orig, dp, tp, pp, gpus) + + return self._items + + def _detect_original_parallelism(self) -> Optional[dict]: + """Detect original DP/TP/PP from LogItem comm groups.""" + dp_sizes = set() + tp_sizes = set() + + for item in self._items: + if item.comm_group == CommGroup.dp: + dp_sizes.add(item.comm_group_size) + elif item.comm_group in ( + CommGroup.tp, + CommGroup.tp_pp, + ): + tp_sizes.add(item.comm_group_size) + + # Heuristic: largest DP group is the DP size, largest TP group is TP size + dp = max(dp_sizes) if dp_sizes else 1 + tp = max(tp_sizes) if tp_sizes else 1 + + if dp == 1 and tp == 1: + return None + + return {"dp": dp, "tp": tp, "pp": 1} + + def _rescale_item( + self, + item: LogItem, + orig: dict, + dp: int, + tp: int, + pp: int, + gpus: int, + ): + """Recalculate communication parameters for a single LogItem.""" + orig_dp = orig["dp"] + orig_tp = orig["tp"] + + # Compute scaling ratios + tp_ratio = tp / orig_tp + dp_ratio = dp / orig_dp + + # TP communication: msg_size scales inversely with tp_size + # (more GPUs in TP group = smaller per-GPU message) + if item.comm_group in (CommGroup.tp, CommGroup.tp_pp): + item.comm_group_size = tp + item.msg_size = item.msg_size / tp_ratio + # More TP groups across the cluster = more TP collectives + # but each is smaller. Recalculate elapsed_time via bandwidth. + if item.elapsed_time is not None: + item.elapsed_time = item.elapsed_time / tp_ratio + + # DP communication: group_size changes, msg_size is unchanged + elif item.comm_group == CommGroup.dp: + item.comm_group_size = dp + # DP AllReduce: same msg_size (gradients), different group + if item.elapsed_time is not None: + # DP communication time scales with log2(dp) for ring + item.elapsed_time = item.elapsed_time * ( + math.log2(max(dp, 2)) / math.log2(max(orig_dp, 2)) + ) + + # DP+TP combined: update both + if item.comm_group == CommGroup.dp: + parts = item.stage.split("_") + for i, part in enumerate(parts): + if part.startswith("dp"): + parts[i] = f"dp{dp}" + elif part.startswith("tp"): + parts[i] = f"tp{tp}" + item.stage = "_".join(parts) diff --git a/aicb/tuning/variability.py b/aicb/tuning/variability.py new file mode 100644 index 000000000..1ca84211d --- /dev/null +++ b/aicb/tuning/variability.py @@ -0,0 +1,111 @@ +"""F003: Variability Injector. + +Standalone Monte Carlo noise injector for LogItem durations. +Can be used independently of TunabilityWrapper for targeted +variability studies. + +Supports: +- Gaussian noise: item.elapsed_time *= (1 + N(0, std_ratio)) +- Uniform noise: item.elapsed_time *= (1 + U(-range, +range)) +- Per-op-type filtering (compute-only, comm-only, all) +""" + +import random +from typing import List, Optional +from log_analyzer.log import LogItem, CommType + + +class VariabilityInjector: + """Adds controlled noise to operation durations. + + Usage: + injector = VariabilityInjector(log_items) + injector.apply_gaussian(std_ratio=0.05, seed=42) + # All op durations now have 5% std Gaussian noise. + """ + + def __init__(self, log_items: List[LogItem]): + self._items = log_items + + def apply_gaussian( + self, + std_ratio: float = 0.05, + op_filter: str = "all", + seed: Optional[int] = None, + min_duration_us: float = 1.0, + ) -> List[LogItem]: + """Apply Gaussian noise: duration *= (1 + N(0, std_ratio)). + + Args: + std_ratio: Std dev as fraction of the duration. + op_filter: "all", "compute", or "comm". + seed: Random seed for reproducibility. + min_duration_us: Floor duration in microseconds. + + Returns: + The modified LogItem list. + """ + if seed is not None: + random.seed(seed) + + for item in self._items: + if not self._matches_filter(item, op_filter): + continue + if item.elapsed_time is None or item.elapsed_time <= 0: + continue + + noise = random.gauss(0.0, std_ratio) + item.elapsed_time = max( + min_duration_us, + item.elapsed_time * (1.0 + noise), + ) + + return self._items + + def apply_uniform( + self, + range_pct: float = 0.10, + op_filter: str = "all", + seed: Optional[int] = None, + min_duration_us: float = 1.0, + ) -> List[LogItem]: + """Apply uniform noise: duration *= (1 + U(-range, +range)). + + Args: + range_pct: Maximum deviation as fraction (e.g., 0.10 = +/-10%). + op_filter: "all", "compute", or "comm". + seed: Random seed for reproducibility. + min_duration_us: Floor duration in microseconds. + + Returns: + The modified LogItem list. + """ + if seed is not None: + random.seed(seed) + + for item in self._items: + if not self._matches_filter(item, op_filter): + continue + if item.elapsed_time is None or item.elapsed_time <= 0: + continue + + noise = random.uniform(-range_pct, +range_pct) + item.elapsed_time = max( + min_duration_us, + item.elapsed_time * (1.0 + noise), + ) + + return self._items + + def _matches_filter(self, item: LogItem, op_filter: str) -> bool: + if op_filter == "all": + return True + is_compute = item.comm_type == CommType.computation + if op_filter == "compute": + return is_compute + if op_filter == "comm": + return not is_compute and item.comm_type not in ( + CommType.epoch_begin, + CommType.epoch_end, + ) + return True diff --git a/aicb/tuning/wrapper.py b/aicb/tuning/wrapper.py new file mode 100644 index 000000000..649708bef --- /dev/null +++ b/aicb/tuning/wrapper.py @@ -0,0 +1,175 @@ +"""F003: Tunability Wrapper (Decorator Pattern). + +Orchestrates straggler injection, workload scaling, and variability +modeling as a decorator layer between mocked_model output and the +workload writer. No changes to core mocked_model logic. + +Design: Non-invasive wrapper that iterates over LogItems after +workload generation. Applies tunability transforms in order: + 1. Scale (recalculate parallelism dimensions) + 2. Inject stragglers (probabilistic delays) + 3. Add variability (duration noise) + +Reference: MLSynth Performance Wrapper pattern (NAIC '25). +""" + +import random +from typing import List, Optional +from log_analyzer.log import LogItem, CommType, CommGroup + + +class TunabilityWrapper: + """Decorator-pattern tunability orchestrator. + + Wraps a list of LogItems (generated workload) and applies + straggler injection, workload scaling, and variability modeling. + + Usage: + items = generate_workload(...) + wrapper = TunabilityWrapper(items) + wrapper.inject_stragglers(gpu_rate=0.02, nic_rate=0.01) + wrapper.add_variability(std_ratio=0.05) + wrapper.apply() + # items now have straggler delays and duration variability + """ + + def __init__(self, log_items: List[LogItem]): + self._items = log_items + self._straggler_config: Optional[dict] = None + self._variability_config: Optional[dict] = None + + # ------------------------------------------------------------------ + # Straggler Injection + # ------------------------------------------------------------------ + + def inject_stragglers( + self, + gpu_rate: float = 0.0, + nic_rate: float = 0.0, + gpu_delay_us: float = 100.0, + nic_delay_us: float = 50.0, + seed: Optional[int] = None, + ) -> "TunabilityWrapper": + """Configure probabilistic straggler delays. + + Args: + gpu_rate: Probability [0,1] that a compute op is delayed. + nic_rate: Probability [0,1] that a comm op is delayed. + gpu_delay_us: Base delay for compute stragglers (microseconds). + nic_delay_us: Base delay for comm stragglers (microseconds). + seed: Random seed for reproducibility. + """ + if seed is not None: + random.seed(seed) + self._straggler_config = { + "gpu_rate": gpu_rate, + "nic_rate": nic_rate, + "gpu_delay_us": gpu_delay_us, + "nic_delay_us": nic_delay_us, + } + return self + + # ------------------------------------------------------------------ + # Variability Modeling + # ------------------------------------------------------------------ + + def add_variability( + self, + std_ratio: float = 0.0, + seed: Optional[int] = None, + ) -> "TunabilityWrapper": + """Configure Monte Carlo duration noise. + + Args: + std_ratio: Std dev as fraction of elapsed_time (e.g., 0.05 = 5%). + seed: Random seed for reproducibility. + """ + if seed is not None: + random.seed(seed) + self._variability_config = { + "std_ratio": std_ratio, + } + return self + + # ------------------------------------------------------------------ + # Apply all configured transforms + # ------------------------------------------------------------------ + + def apply(self) -> List[LogItem]: + """Apply all configured tunability transforms in order. + + Returns: + The modified LogItem list (same reference as input). + """ + # 1. Straggler injection + if self._straggler_config: + self._apply_stragglers() + + # 2. Variability noise + if self._variability_config: + self._apply_variability() + + return self._items + + def _apply_stragglers(self): + cfg = self._straggler_config + for item in self._items: + if item.elapsed_time is None: + continue + + # Compute ops (comp_type) vs communication ops + if item.comm_type == CommType.computation: + if random.random() < cfg["gpu_rate"]: + delay = cfg["gpu_delay_us"] * random.uniform(0.5, 2.0) + item.straggler_delay_us = delay + elif item.comm_type not in ( + CommType.computation, + CommType.epoch_begin, + CommType.epoch_end, + ): + if random.random() < cfg["nic_rate"]: + delay = cfg["nic_delay_us"] * random.uniform(0.5, 2.0) + item.straggler_delay_us = delay + + def _apply_variability(self): + cfg = self._variability_config + for item in self._items: + if item.elapsed_time is not None and item.elapsed_time > 0: + noise = random.gauss(0, cfg["std_ratio"]) + item.elapsed_time = max( + 0, + item.elapsed_time * (1.0 + noise), + ) + + # ------------------------------------------------------------------ + # Chained scaling support (delegates to WorkloadScaler) + # ------------------------------------------------------------------ + + def scale( + self, + new_dp: Optional[int] = None, + new_tp: Optional[int] = None, + new_pp: Optional[int] = None, + total_gpus: Optional[int] = None, + ) -> "TunabilityWrapper": + """Scale workload to new parallelism dimensions. + + Delegates to WorkloadScaler. Changes communication topology + without regenerating layer structure. + + Args: + new_dp: New data-parallel size. + new_tp: New tensor-parallel size. + new_pp: New pipeline-parallel size. + total_gpus: Total GPU count (auto-computed if not given). + """ + from .scaler import WorkloadScaler + + scaler = WorkloadScaler(self._items) + scaler.scale( + new_dp=new_dp, + new_tp=new_tp, + new_pp=new_pp, + total_gpus=total_gpus, + ) + return self diff --git a/aicb/utils/benchmark_logger.py b/aicb/utils/benchmark_logger.py new file mode 100755 index 000000000..f1071b4df --- /dev/null +++ b/aicb/utils/benchmark_logger.py @@ -0,0 +1,106 @@ +""" +Copyright (c) 2021, Alibaba Group; +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import sys +import torch +import logging +from utils.timer import Timer +from log_analyzer.log import Log, LogItem + + +class LoggerFactory: + + @staticmethod + def create_logger(name=None, level=logging.INFO): + """create a logger + + Args: + name (str): name of the logger + level: level of logger + + Raises: + ValueError is name is None + """ + + if name is None: + raise ValueError("name for logger cannot be None") + + formatter = logging.Formatter("[%(asctime)s] [%(levelname)s] %(message)s") + + logger_ = logging.getLogger(name) + logger_.setLevel(level) + logger_.propagate = False + ch = logging.StreamHandler(stream=sys.stdout) + ch.setLevel(level) + ch.setFormatter(formatter) + logger_.addHandler(ch) + return logger_ + + +logger = LoggerFactory.create_logger(name="LLM_Comm_Benchmark", level=logging.INFO) + + +class BenchLogger: + def __init__(self): + self.comm_log = Log() + self.enable = True + self.timer = Timer() + self.epoch_timer = Timer(use_host_timer=True) + self.epoch = 0 + self.epoch_timer.start() + + def log_timing(self, name): + def decorator(func): + def wrapper(*args, **kwargs): + self.timer.start() + result = func(*args, **kwargs) + elapsed_time_ms = self.timer.stop() + + log_item = next((item for item in args if isinstance(item, LogItem))) + if log_item.additional == 'overlap': + log_item.elapsed_time = 0 + else: + log_item.elapsed_time = elapsed_time_ms + self.comm_log.add_comm_log(log_item) + if torch.distributed.get_rank() == 0: + logger.info(log_item.view_as_ds_log()) + return result + + return wrapper + + return decorator + + def end_epoch(self, log_item): + torch.cuda.synchronize() + elapsed_time_ms = self.epoch_timer.stop() + if torch.distributed.get_rank() == 0: + logger.info( + f"[RANK 0] --------epoch {self.epoch} | micro_step time {elapsed_time_ms:.2f} ---------\n" + ) + log_item.elapsed_time = elapsed_time_ms + self.comm_log.add_comm_log(log_item) + self.epoch += 1 + self.epoch_timer.start() + + def dump_log(self, filename): + csv_filename = self.comm_log.dump(filename) + return csv_filename + + def analyze_comm_log(self, print_fn=logger.info): + return self.comm_log.analyze(print_fn) + + def analyze_comm_time(self, print_fn=logger.info): + return self.comm_log.analyze_time(print_fn) + + +bench_logger = BenchLogger() diff --git a/aicb/utils/deepgemm_utils.py b/aicb/utils/deepgemm_utils.py new file mode 100644 index 000000000..0cccc0a76 --- /dev/null +++ b/aicb/utils/deepgemm_utils.py @@ -0,0 +1,235 @@ +import torch +from typing import Tuple, Generator +import deep_gemm +import enum +import random +from deep_gemm.testing import bench_kineto, calc_diff, count_bytes +from deep_gemm.utils import (align, ceil_div, + get_mk_alignment_for_contiguous_layout, + per_block_cast_to_fp8, per_token_cast_to_fp8) +# from deep_gemm.jit_kernels.utils import get_m_alignment_for_contiguous_layout + +# From DeepGEMM/tests/generators.py +class KernelType(enum.Enum): + Kernel1D1D = 0 + Kernel1D2D = 1 + KernelNoSF = 2 + + def is_1d1d(self): + return self.value == 0 + + def is_1d2d(self): + return self.value == 1 + + def is_nosf(self): + return self.value == 2 + + +class MajorTypeAB(enum.Enum): + KMajor = 0 + MNMajor = 1 + + def is_k_major(self): + return self.value == 0 + + def is_mn_major(self): + return self.value == 1 + + +# === Common === +def generate_normal( + m: int, + n: int, + k: int, + major_a: MajorTypeAB, + major_b: MajorTypeAB, + accumulate: bool, + out_dtype: torch.dtype, + kernel_type: KernelType, + use_ue8m0: bool = False, + use_bf16: bool = False, +): + a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) + b = torch.randn((n, k), device="cuda", dtype=torch.bfloat16) + d = ( + torch.randn((m, n), device="cuda", dtype=out_dtype) * 32 + if accumulate + else torch.empty((m, n), device="cuda", dtype=out_dtype) + ) + c = d if accumulate else None + ref_d = (a.float() @ b.float().t() + (c if accumulate else 0)).to(out_dtype) + + if use_bf16: + a = a if major_a.is_k_major() else a.T.contiguous().T + b = b if major_b.is_k_major() else b.T.contiguous().T + return a, b, c, d, ref_d + + a_fp8 = per_token_cast_to_fp8(a, use_ue8m0=use_ue8m0) + b_fp8 = ( + per_token_cast_to_fp8(b, use_ue8m0=use_ue8m0) + if kernel_type.is_1d1d() and accumulate + else per_block_cast_to_fp8(b, use_ue8m0=use_ue8m0) + ) + a_fp8 = a_fp8 if major_a.is_k_major() else (a_fp8[0].T.contiguous().T, a_fp8[1]) + b_fp8 = b_fp8 if major_b.is_k_major() else (b_fp8[0].T.contiguous().T, b_fp8[1]) + return a_fp8, b_fp8, c, d, ref_d + + +# === Masked === +def enumerate_m_grouped_masked() -> Generator: + max_m = 128#4096 + yield KernelType.Kernel1D2D, max_m + +def generate_m_grouped_masked( + num_groups: int, + max_m: int, + expected_m_per_group: int, + n: int, + k: int, + use_ue8m0: bool = False, + use_bf16: bool = False, +): + a = torch.randn((num_groups, max_m, k), device="cuda", dtype=torch.bfloat16) + b = torch.randn((num_groups, n, k), device="cuda", dtype=torch.bfloat16) + d = torch.empty((num_groups, max_m, n), device="cuda", dtype=torch.bfloat16) + ref_d = torch.einsum("gmk,gnk->gmn", a, b) + + masked_m = torch.empty((num_groups,), device="cuda", dtype=torch.int) + for j in range(num_groups): + masked_m[j] = int(expected_m_per_group * random.uniform(0.7, 1.3)) + assert masked_m.amax().item() <= max_m + + if use_bf16: + return a, b, masked_m, d, ref_d + + a_fp8 = ( + torch.empty_like(a, dtype=torch.float8_e4m3fn), + torch.empty( + (num_groups, max_m, ceil_div(k, 128)), device="cuda", dtype=torch.float + ), + ) + b_fp8 = ( + torch.empty_like(b, dtype=torch.float8_e4m3fn), + torch.empty( + (num_groups, ceil_div(n, 128), ceil_div(k, 128)), + device="cuda", + dtype=torch.float, + ), + ) + for i in range(num_groups): + a_fp8[0][i], a_fp8[1][i] = per_token_cast_to_fp8(a[i], use_ue8m0=use_ue8m0) + b_fp8[0][i], b_fp8[1][i] = per_block_cast_to_fp8(b[i], use_ue8m0=use_ue8m0) + + return a_fp8, b_fp8, masked_m, d, ref_d + + +# === Contiguous === +def enumerate_m_grouped_contiguous() -> Generator: + yield KernelType.Kernel1D2D, MajorTypeAB.KMajor, MajorTypeAB.KMajor + +def generate_m_grouped_contiguous( + num_groups: int, + expected_m_per_group: int, + n: int, + k: int, + major_a: MajorTypeAB, + major_b: MajorTypeAB, + use_ue8m0: bool = False, + use_bf16: bool = False, +): + actual_ms = [int(expected_m_per_group * random.uniform(0.7, 1.3)) for _ in range(num_groups)] + aligned_ms = [align(actual_m, get_mk_alignment_for_contiguous_layout()) for actual_m in actual_ms] + m = sum(aligned_ms) + + a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) + b = torch.randn((num_groups, n, k), device="cuda", dtype=torch.bfloat16) + m_indices = torch.empty(m, device="cuda", dtype=torch.int32) + d = torch.empty((m, n), device="cuda", dtype=torch.bfloat16) + ref_d = torch.randn((m, n), device="cuda", dtype=torch.bfloat16) + + start = 0 + for i, (actual_m, aligned_m) in enumerate(zip(actual_ms, aligned_ms)): + actual_end = start + actual_m + aligned_end = start + aligned_m + m_indices[start:actual_end] = i + m_indices[actual_end:aligned_end] = -1 + ref_d[start:aligned_end] = a[start:aligned_end] @ b[i].t() + start = aligned_end + ref_d = torch.where((m_indices == -1).unsqueeze(1), torch.zeros_like(ref_d), ref_d) + + if use_bf16: + b = b if major_b.is_k_major() else b.mT.contiguous().mT + return m, a, b, m_indices, d, ref_d + + assert major_a.is_k_major() + a_fp8 = per_token_cast_to_fp8(a, use_ue8m0=use_ue8m0) + b_fp8 = ( + torch.empty_like(b, dtype=torch.float8_e4m3fn), + torch.empty( + (num_groups, ceil_div(n, 128), ceil_div(k, 128)), + device="cuda", + dtype=torch.float, + ), + ) + for i in range(num_groups): + b_fp8[0][i], b_fp8[1][i] = per_block_cast_to_fp8(b[i], use_ue8m0=use_ue8m0) + b_fp8 = b_fp8 if major_b.is_k_major() else (b_fp8[0].mT.contiguous().mT, b_fp8[1]) + return m, a_fp8, b_fp8, m_indices, d, ref_d + +# This function uses generate_normal to construct x_fp8, y_fp8, out, ref_out +def construct(m: int, k: int, n: int) -> \ + Tuple[Tuple[torch.Tensor, torch.Tensor], Tuple[torch.Tensor, torch.Tensor], torch.Tensor, torch.Tensor]: + a_fp8, b_fp8, c, d, ref_d = generate_normal(m, n, k, MajorTypeAB.KMajor, MajorTypeAB.KMajor, False, torch.bfloat16, KernelType.Kernel1D2D) + return a_fp8, b_fp8, c, d, ref_d + +# This function uses generate_m_grouped_masked and tests m_grouped_fp8_gemm_nt_masked +def test_func_masked(num_groups, expected_m_per_group, k, n) -> None: + [(kerneltype, max_m)] = enumerate_m_grouped_masked() + a_fp8, b_fp8, masked_m, d, ref_d = generate_m_grouped_masked( + num_groups, max_m, expected_m_per_group, n, k + ) + deep_gemm.m_grouped_fp8_gemm_nt_masked( + a_fp8, + b_fp8, + d, + masked_m, + expected_m_per_group, + disable_ue8m0_cast=True, + ) + +# This function uses generate_m_grouped_contiguous and tests m_grouped_fp8_gemm_nt_contiguous +def test_func_contiguous(num_groups, expected_m_per_group, k, n) -> None: + [(kerneltype, major_a, major_b)] = enumerate_m_grouped_contiguous() + m, a_fp8, b_fp8, m_indices, d, ref_d = generate_m_grouped_contiguous( + num_groups, expected_m_per_group, n, k, major_a, major_b + ) + deep_gemm.m_grouped_fp8_gemm_nt_contiguous( + a_fp8, b_fp8, d, m_indices, disable_ue8m0_cast=True + ) + +def bench_masked(num_groups, expected_m_per_group, k, n) -> float: + [(kerneltype, max_m)] = enumerate_m_grouped_masked() + a_fp8, b_fp8, masked_m, d, ref_d = generate_m_grouped_masked( + num_groups, max_m, expected_m_per_group, n, k + ) + def test_func(): + deep_gemm.m_grouped_fp8_gemm_nt_masked( + a_fp8, + b_fp8, + d, + masked_m, + expected_m_per_group, + disable_ue8m0_cast=True, + ) + return bench_kineto(test_func, "fp8_gemm", suppress_kineto_output=True) + +def bench_contiguous(num_groups, expected_m_per_group, k, n) -> float: + [(kerneltype, major_a, major_b)] = enumerate_m_grouped_contiguous() + m, a_fp8, b_fp8, m_indices, d, ref_d = generate_m_grouped_contiguous( + num_groups, expected_m_per_group, n, k, major_a, major_b + ) + def test_func(): + deep_gemm.m_grouped_fp8_gemm_nt_contiguous( + a_fp8, b_fp8, d, m_indices, disable_ue8m0_cast=True + ) + return bench_kineto(test_func, "fp8_gemm", suppress_kineto_output=True) diff --git a/aicb/utils/rank_analysis.csv b/aicb/utils/rank_analysis.csv new file mode 100644 index 000000000..ff6f5d870 --- /dev/null +++ b/aicb/utils/rank_analysis.csv @@ -0,0 +1,9 @@ +Rank ID,Total Probability +0,0.219079 +1,0.154441 +2,0.087829 +3,0.196711 +4,0.088651 +5,0.066118 +6,0.045559 +7,0.141612 diff --git a/aicb/utils/rank_mapper.py b/aicb/utils/rank_mapper.py new file mode 100644 index 000000000..207161ff0 --- /dev/null +++ b/aicb/utils/rank_mapper.py @@ -0,0 +1,70 @@ +"""CommGroup-to-RankGenerator mapping utility for communication domain (rank) information. + +This module bridges the CommGroup enum values to RankGenerator.get_ranks() tokens, +providing the actual GPU rank IDs participating in each communication operation. +""" + +import logging +from typing import List, Dict + +from utils.utils import CommGroup, RankGenerator + +logger = logging.getLogger(__name__) + +_COMM_GROUP_TOKEN_MAP = { + CommGroup.tp_group: ("tp", False), + CommGroup.pp_group: ("pp", False), + CommGroup.dp_group: ("dp", False), + CommGroup.ep_group: ("ep", True), + CommGroup.ep_dp_group: ("ep-dp", True), + CommGroup.ep_tp_group: ("ep-tp", True), + CommGroup.embedding_group: ("tp", False), +} + + +def get_rank_list_for_comm_group( + rank_generator: RankGenerator, + comm_group: CommGroup, + comm_group_size: int = None, + ref_rank: int = 0, +) -> List[int]: + if comm_group is None: + return [] + if comm_group == CommGroup.all: + return list(range(rank_generator.world_size)) + if comm_group not in _COMM_GROUP_TOKEN_MAP: + raise ValueError(f"Unknown CommGroup value: {comm_group}") + token, independent_ep = _COMM_GROUP_TOKEN_MAP[comm_group] + groups = rank_generator.get_ranks(token, independent_ep=independent_ep) + for group in groups: + if ref_rank in group: + return group + raise ValueError( + f"Rank {ref_rank} not found in any {comm_group} group. Groups: {groups[:3]}..." + ) + + +def build_rank_mapping_table(rank_generator: RankGenerator) -> List[Dict]: + rows = [] + for comm_group, (token, independent_ep) in sorted( + _COMM_GROUP_TOKEN_MAP.items(), key=lambda x: x[0].value + ): + try: + groups = rank_generator.get_ranks(token, independent_ep=independent_ep) + group_size = len(groups[0]) if groups else 0 + rank_groups_str = " ".join( + "[" + ",".join(str(r) for r in g) + "]" for g in groups + ) + rows.append({ + "group": comm_group.value, + "size": group_size, + "rank_groups": rank_groups_str, + }) + except Exception as e: + logger.warning(f"Skipping {comm_group.value} in mapping table: {e}") + rows.append({ + "group": CommGroup.all.value, + "size": rank_generator.world_size, + "rank_groups": "[" + ",".join(str(r) for r in range(rank_generator.world_size)) + "]", + }) + return rows diff --git a/aicb/utils/timer.py b/aicb/utils/timer.py new file mode 100644 index 000000000..3c40a9525 --- /dev/null +++ b/aicb/utils/timer.py @@ -0,0 +1,59 @@ +""" +Copyright (c) 2021, Alibaba Group; +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import time +import torch + + +class CudaEventTimer(object): + """Borrowed from Deepspeed""" + def __init__(self, start_event: torch.cuda.Event, end_event: torch.cuda.Event): + self.start_event = start_event + self.end_event = end_event + + def get_elapsed_msec(self): + #torch.cuda.current_stream().wait_event(self.end_event) + self.end_event.synchronize() + return self.start_event.elapsed_time(self.end_event) + + +class Timer: + def __init__(self, use_host_timer=False): + self.started_ = False + self.use_host_timer = use_host_timer + self.start_event = None + self.start_time = 0.0 + + def start(self): + """Start the timer.""" + assert not self.started_, f"timer has already been started" + if self.use_host_timer: + self.start_time = time.time() + else: + self.start_event = torch.cuda.Event(enable_timing=True) + self.start_event.record() + self.started_ = True + + def stop(self): + """Stop the timer.""" + assert self.started_, "timer is not started" + self.started_ = False + if self.use_host_timer: + end_time = time.time() + return (end_time - self.start_time) * 1000 + else: + end_event = torch.cuda.Event(enable_timing=True) + end_event.record() + event_timer = CudaEventTimer(self.start_event, end_event) + self.start_event = None + return event_timer.get_elapsed_msec() diff --git a/aicb/utils/utils.py b/aicb/utils/utils.py new file mode 100755 index 000000000..223bf6a33 --- /dev/null +++ b/aicb/utils/utils.py @@ -0,0 +1,952 @@ +""" +Copyright (c) 2021, Alibaba Group; +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from typing import List, Dict, Tuple +import pandas as pd +import pickle +from enum import Enum +import argparse +import sys +import time +import os +import json +from collections import defaultdict +import math +import re +import numpy as np + +try: + import torch +except ImportError as e: + torch = None + print("Failed to import 'torch'.") + +def generate_masked_orthogonal_rank_groups( + world_size: int, parallel_size: List[int], mask: List[bool], +) -> List[List[int]]: + r"""Generate orthogonal parallel groups based on the parallel size and mask. + + Arguments: + world_size (int): world size + + parallel_size (List[int]): + The parallel size of each orthogonal parallel type. For example, if + tensor_parallel_size = 2, pipeline_model_parallel_group = 3, data_parallel_size = 4, + and the parallel mapping order is tp-pp-dp, then the parallel_size = [2, 3, 4]. + + mask (List[bool]): + The mask controls which parallel methods the generated groups represent. If mask[i] is + True, it means the generated group contains the i-th parallelism method. For example, + if parallel_size = [tp_size, pp_size, dp_size], and mask = [True, False , True], then + the generated group is the `tp-dp` group, if the mask = [False, True, False], then the + generated group is the `pp` group. + + Algorithm: + For orthogonal parallelism, such as tp/dp/pp/cp, the global_rank and + local_rank satisfy the following equation: + global_rank = tp_rank + dp_rank * tp_size + pp_rank * tp_size * dp_size (1) + tp_rank \in [0, tp_size) + dp_rank \in [0, dp_size) + pp_rank \in [0, pp_size) + + If we want to get the `dp_group` (tp_size * pp_size groups of dp_size ranks each. + For example, if the gpu size is 8 and order is 'tp-pp-dp', size is '2-2-2', and the + dp_group here is [[0, 4], [1, 5], [2, 6], [3, 7]].) + The tp_rank and pp_rank will be combined to form the `dp_group_index`. + dp_group_index = tp_rank + pp_rank * tp_size (2) + + So, Given that tp_rank and pp_rank satisfy equation (2), and dp_rank in + range(0, dp_size), the ranks in dp_group[dp_group_index] satisfies the + equation (1). + + This function solve this math problem. + + For example, if the parallel_size = [tp_size, dp_size, pp_size] = [2, 3, 4], + and the mask = [False, True, False]. Then, + dp_group_index(0) = tp_rank(0) + pp_rank(0) * 2 + dp_group_index(1) = tp_rank(1) + pp_rank(0) * 2 + ... + dp_group_index(7) = tp_rank(1) + pp_rank(3) * 2 + + dp_group[0] = 0 + range(0, 3) * 2 + 0 = [0, 2, 4] + dp_group[1] = 1 + range(0, 3) * 2 + 0 = [1, 3, 5] + ... + dp_group[7] = 1 + range(0, 3) * 2 + 3 * 2 * 3 = [19, 21, 23] + """ + + def prefix_product(a: List[int], init=1) -> List[int]: + r = [init] + for v in a: + init = init * v + r.append(init) + return r + + def inner_product(a: List[int], b: List[int]) -> int: + return sum([x * y for x, y in zip(a, b)]) + + def decompose(index, shape, stride=None): + ''' + This function solve the math problem below: + There is an equation: + index = sum(idx[i] * stride[i]) + And given the value of index, stride. + Return the idx. + This function will used to get the pp/dp/pp_rank + from group_index and rank_in_group. + ''' + if stride is None: + stride = prefix_product(shape) + idx = [(index // d) % s for s, d in zip(shape, stride)] + # stride is a prefix_product result. And the value of stride[-1] + # is not used. + assert ( + sum([x * y for x, y in zip(idx, stride[:-1])]) == index + ), "idx {} with shape {} mismatch the return idx {}".format(index, shape, idx) + return idx + + masked_shape = [s for s, m in zip(parallel_size, mask) if m] + unmasked_shape = [s for s, m in zip(parallel_size, mask) if not m] + + global_stride = prefix_product(parallel_size) + masked_stride = [d for d, m in zip(global_stride, mask) if m] + unmasked_stride = [d for d, m in zip(global_stride, mask) if not m] + + group_size = prefix_product(masked_shape)[-1] + num_of_group = world_size // group_size + + ranks = [] + for group_index in range(num_of_group): + # get indices from unmaksed for group_index. + decomposed_group_idx = decompose(group_index, unmasked_shape) + rank = [] + for rank_in_group in range(group_size): + # get indices from masked for rank_in_group. + decomposed_rank_idx = decompose(rank_in_group, masked_shape) + rank.append( + inner_product(decomposed_rank_idx, masked_stride) + + inner_product(decomposed_group_idx, unmasked_stride) + ) + ranks.append(rank) + return ranks +class RankGenerator(object): + def __init__(self, tp: int, ep: int, dp: int, pp: int, cp: int, order: str) -> None: + self.tp = tp + self.ep = ep + self.dp = dp + self.pp = pp + self.cp = cp + self.world_size = tp * dp * pp * cp + + self.name_to_size = { + "tp": self.tp, + "pp": self.pp, + "dp": self.dp, + "ep": self.ep, + "cp": self.cp, + } + self.order = order + order = order.lower() + + if 'ep' in order: + if 'ep-dp' not in order and 'dp-ep' not in order: + raise RuntimeError(f"The ep and dp must be adjacent in order ({self.order}).") + + for name in self.name_to_size.keys(): + if name not in order and self.name_to_size[name] != 1: + raise RuntimeError( + f"The size of ({name}) is ({self.name_to_size[name]}), but you haven't specified the order ({self.order})." + ) + elif name not in order: + order = order + '-' + name + + self.order_w_ep = order + self.order_wo_ep = '-'.join([token for token in order.split('-') if token != 'ep']) + self.ordered_size_wo_ep = [] + self.ordered_size_w_ep = [] + + for token in order.split('-'): + if token == 'dp': + self.ordered_size_w_ep.append(self.dp // self.ep) + self.ordered_size_wo_ep.append(self.dp) + elif token == 'ep': + self.ordered_size_w_ep.append(self.ep) + else: + self.ordered_size_w_ep.append(self.name_to_size[token]) + self.ordered_size_wo_ep.append(self.name_to_size[token]) + + def get_mask(self, order: str, token: str): + ordered_token = order.split('-') + token = token.split('-') + mask = [False] * len(ordered_token) + for t in token: + mask[ordered_token.index(t)] = True + return mask + + def get_ranks(self, token, independent_ep=False): + '''Get rank group by input token. + + Arguments: + token (str): + Specify the ranks type that want to get. If we want + to obtain multiple parallel types, we can use a hyphen + '-' to separate them. For example, if we want to obtain + the TP_DP group, the token should be 'tp-dp'. + + independent_ep (bool: True): + This flag controls whether we treat EP and DP independently. + EP shares ranks with DP, if we want to get ranks related to + EP, we should set the flag. For example, get_ranks('dp', True) + will get DP modulo EP group, and get_ranks('dp', False) will + get full DP group. + ''' + if independent_ep: + parallel_size = self.ordered_size_w_ep + order = self.order_w_ep + else: + parallel_size = self.ordered_size_wo_ep + order = self.order_wo_ep + mask = self.get_mask(order, token) + ranks = generate_masked_orthogonal_rank_groups(self.world_size, parallel_size, mask) + return ranks + +def gelu_impl(x): + """OpenAI's gelu implementation.""" + return ( + 0.5 * x * (1.0 + torch.tanh(0.7978845608028654 * x * (1.0 + 0.044715 * x * x))) + ) + + +def openai_gelu(x): + return gelu_impl(x) + + +def erf_gelu(x): + return ( + x + * 0.5 + * ( + torch.erf(x / 1.41421).to(dtype=x.dtype) + + torch.ones_like(x).to(dtype=x.dtype) + ) + ) + + +def Comp_with_aiob(workload, compute_cache): + for item in workload.workload: + if item.comm_type == CommType.computation: + for key in compute_cache: + key_temp = key.split("_")[0] + if key_temp in item.stage: + item._elapsed_time = compute_cache[key] + break + return workload + + +def get_comp_out(args): + vocab_size = args.vocab_size + batch_size = args.micro_batch + seq_len = args.seq_length + tp = args.tensor_model_parallel_size + vocab_size = args.padded_vocab_size + if "Megatron" or "DeepSeek" in args.frame: + device = torch.cuda.current_device() + from workload_generator.mocked_model.training.AiobMegatron import MegatronModel + measure_model = MegatronModel(args) + measure_model.train() + if args.dtype == "bfloat16": + dtype = torch.bfloat16 + elif args.dtype == "float16": + dtype = torch.float16 + else: + dtype = torch.float32 + # total_input_1 = torch.rand(args.seq_len, + # args.batch_size, + # args.hidden_size, + # device=device).to(dtype) + masked_input = torch.randint( + 0, + math.ceil(vocab_size / tp), + (batch_size, seq_len), + device=device, + dtype=torch.int64, + ) + filepath = measure_model(masked_input) + return filepath + + + +def extract_inference_averages(file_path,args): + if file_path is None or file_path == "": + return {} + attention_norm_avg_sum = 0.0 + attention_gdn_avg_sum = 0.0 + attention_avg_sum = 0.0 + mlp_avg_sum = 0.0 + moe_norm_avg_sum = 0.0 + moe_route_avg_sum = 0.0 + moe_expert_sum = 0.0 + + section_header_re = re.compile(r"^(\w+):") + time_gpu_avg_re = re.compile(r"time_gpu_avg:\s+(\d+(\.\d+)?)") + # time_gpu_min_re = re.compile(r"time_gpu_min:\s+(\d+(\.\d+)?)") + + with open(file_path, "r") as file: + current_section = None + + for line in file: + header_match = section_header_re.match(line) + if header_match: + current_section = header_match.group(1).strip() + + avg_match = time_gpu_avg_re.search(line) + # min_match = time_gpu_min_re.search(line) + if avg_match and current_section: + avg_value = float(avg_match.group(1)) * 1000 + if "atten_norm" in current_section: + attention_norm_avg_sum += avg_value + elif "gdn" in current_section: + attention_gdn_avg_sum += avg_value + elif "atten" in current_section: + attention_avg_sum += avg_value + elif "mlp" in current_section: + mlp_avg_sum += avg_value + elif "moe_norm" in current_section: + moe_norm_avg_sum += avg_value + elif "moe_route" in current_section: + moe_route_avg_sum += avg_value + elif "moe" in current_section: + moe_expert_sum += avg_value + + full_cache = { + "attention_norm": round(attention_norm_avg_sum), + "attention_gdn": round(attention_gdn_avg_sum), + "attention_layer": round(attention_avg_sum), + "mlp": round(mlp_avg_sum), + "moe_norm": round(moe_norm_avg_sum), + "moe_route": round(moe_route_avg_sum), + "moe_expert": round(moe_expert_sum), + } + compute_cache = {key: value for key, value in full_cache.items() if value != 0} + + return compute_cache + + +def get_compute_path(args): + result_dir = "./results/aiob_outputs" + if not os.path.isdir(result_dir): + os.makedirs(result_dir) + phase = args.phase + filename = f"{args.model_name}-world_size{args.world_size}-tp{args.tensor_model_parallel_size}-pp1-ep{args.expert_model_parallel_size}-bpg{args.micro_batch}-seq{args.seq_length}-{phase}.txt" + filepath = os.path.join(result_dir, filename) + return filepath + +def write_time(time_list, args): + filepath = get_compute_path(args) + with open(filepath, "w") as file: + file.write(f"inference_term:{args.phase}\n") + data_str = json.dumps(time_list, indent=4) + + file.write(data_str) + return filepath + +def calculate_stats(time_list, file_name): + stats = {} + for key, values in time_list.items(): + times = [entry['time_gpu'] for entry in values] + variance = np.var(times) + p99 = np.percentile(times, 99) + stats[key] = {'variance': variance, 'P99': p99} + with open(file_name, 'w') as file: + json.dump(stats, file, indent=4) + print("output stats successfully") + +def extract_averages(file_path,args): + attention_avg_sum = 0.0 + mlp_avg_sum = 0.0 + other_avgs = {} + grad_forward = 0.0 + grad_backward = 0.0 + + section_header_re = re.compile(r"^(\w+):") + time_gpu_avg_re = re.compile(r"time_gpu_avg:\s+(\d+(\.\d+)?)") + time_gpu_min_re = re.compile(r"time_gpu_min:\s+(\d+(\.\d+)?)") + + # from DeepSeek's MLA per-layers compute + # must match with that of AiobDeepSeek->DeepSeekMLA's return from forward() + # FIXME: this should be more generic + per_layer_time_map = { + "attention_linear_q_lora": 0, + "attention_q_column": 0, + "attention_linear_kv_lora": 0, + "attention_kv_column": 0, + "attention_o_row": 0 + } + + with open(file_path, "r") as file: + current_section = None + + for line in file: + header_match = section_header_re.match(line) + if header_match: + current_section = header_match.group(1).strip() + + avg_match = time_gpu_avg_re.search(line) + min_match = time_gpu_min_re.search(line) + if current_section == "param_time": + if min_match: + grad_forward = float(min_match.group(1)) * 1000 #us + if avg_match: + grad_backward = float(avg_match.group(1)) * 1000 + elif avg_match and current_section: + avg_value = float(avg_match.group(1)) * 1000 + if "atten" in current_section or current_section == "layernorm": + + if args.recompute_activations and 'flash' in current_section: + attention_avg_sum += avg_value*2 + else: + attention_avg_sum += avg_value + elif "mlp" in current_section or current_section == "layernorm2": + mlp_avg_sum += avg_value + else: + other_avgs[current_section] = avg_value + + if current_section in per_layer_time_map.keys(): + per_layer_time_map[current_section] = round(avg_value) + + + # 四舍五入并转换为整数 + attention_forward = round(attention_avg_sum) + attention_backward = attention_forward + mlp_forward = round(mlp_avg_sum) + mlp_backward = mlp_forward + grad_backward = round(grad_backward) + grad_forward = round(grad_forward) + other_avgs_int = {k: round(v) for k, v in other_avgs.items() if k != "param_time"} + + a100_compute_cache = { + "attention_forward": attention_forward, + "attention_backward": attention_backward, + "mlp_forward": mlp_forward, + "mlp_backward": mlp_backward, + "grad_forward": grad_forward, + "grad_backward": grad_backward, + } + a100_compute_cache.update(other_avgs_int) + a100_compute_cache.update(per_layer_time_map) + + return a100_compute_cache + + +def process_all_keys(input_file): + + with open(input_file, "r") as file: + first_line_str = file.readline().strip() + remaining_content = file.read().strip() + # 尝试修复和构建合法的 JSON 字符串 + corrected_content = remaining_content.replace("}{", "},{").replace("]}{", "]},{") + + # 构建 JSON 数组 + json_str = f"[{corrected_content}]" + + try: + data = json.loads(json_str) + + processed_results = defaultdict(lambda: defaultdict(list)) + for entry in data: + for key, measurements in entry.items(): + for measure in measurements: + measure_key, measure_value = next(iter(measure.items())) + if "time_gpu" in measure_key: + processed_results[key]["time_gpu"].append(measure["time_gpu"]) + else: + processed_results[key][measure_key] = measure_value + + for key, values in processed_results.items(): + if "time_gpu" in values: + gpu_times = values["time_gpu"] + min_time_gpu = min(gpu_times) + gpu_times_filtered = [t for t in gpu_times if t <= 3 * min_time_gpu] + values["time_gpu_max"] = max(gpu_times_filtered) + values["time_gpu_min"] = min_time_gpu + values["time_gpu_avg"] = sum(gpu_times_filtered) / len( + gpu_times_filtered + ) + del values["time_gpu"] + + with open(input_file, "w") as outfile: + outfile.write(first_line_str + "\n") + for key, values in processed_results.items(): + outfile.write(f"{key}:\n") + for subkey, subvalue in values.items(): + outfile.write(f" {subkey}: {subvalue}\n") + print(f"Compute-results save in:{input_file}") + + except json.JSONDecodeError as e: + print(f"Failed to decode JSON: {e}") + print("Invalid JSON content:\n", corrected_content) + + +def cuda_timing_decorator(func): + def wrapper(*args, **kwargs): + + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + + start_event.record() + result = func(*args, **kwargs) + end_event.record() + torch.cuda.synchronize() + + elapsed_time_ms = start_event.elapsed_time(end_event) * 1000 # 时间以毫秒为单位 + return result, elapsed_time_ms + + return wrapper +def get_aiob_path(args): + result_dir = "./results/aiob_outputs" + if not os.path.isdir(result_dir): + os.makedirs(result_dir) + filename = f"{args.model_name}-world_size{args.world_size}-tp{args.tensor_model_parallel_size}-pp{args.pipeline_model_parallel}-ep{args.expert_model_parallel_size}-gbs{args.global_batch}-mbs{args.micro_batch}-seq{args.seq_length}-flash_attn-{args.use_flash_attn}.txt" + filepath = os.path.join(result_dir, filename) + return filepath + +def write_op(time_list, args): + filepath = get_aiob_path(args) + with open(filepath, "w") as file: + file.write(f"train_iter:{args.epoch_num}\n") + data_str = json.dumps(time_list, indent=4) + + file.write(data_str) + return filepath + + +class ReduceOp(Enum): + SUM = 0 + PRODUCT = 1 + MIN = 2 + MAX = 3 + BAND = 4 + BOR = 5 + BXOR = 6 + AVG = 7 + UNUSED = 8 + + +class CommType(str, Enum): + """Enum class for possible comm types""" + + all_reduce = "all_reduce" + isend = "isend" + irecv = "irecv" + broadcast = "broadcast" + all_gather = "all_gather" + reduce_scatter = "reduce_scatter" + barrier = "barrier" + reduce = "reduce" + reduce_scatter_tensor = "reduce_scatter_tensor" + all_gather_into_tensor = "all_gather_into_tensor" + computation = "computation" + epoch_end = "epoch_end" + all_to_all = "all_to_all" + + @classmethod + def get_comm_type(cls, value): + for comm_type in cls: + if comm_type.value == value: + return comm_type + raise ValueError("Invailid communication type") + + +class CommGroup(str, Enum): + """Enum class for possible comm groups""" + + dp_group = "dp_group" + pp_group = "pp_group" + tp_group = "tp_group" + ep_group = "ep_group" + ep_dp_group = "ep_dp_group" + ep_tp_group = "ep_tp_group" + embedding_group = "embedding_group" + all = "all_nodes" + + +class WorkloadWriter: + @staticmethod + def write_workload(workload: List[Dict], args, filename: str): + df = pd.DataFrame.from_dict(workload) + df = df.fillna(-1) + df.to_csv(filename, index=False) + + @staticmethod + def load_workload(filename: str) -> List[Dict]: + filename = filename.split(".") + filename[-1] = "pkl" + filename = ".".join(filename) + workload, args = pickle.load(open(filename, "rb")) + return workload, args + + +def get_params(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--frame", + help="communication framework", + choices=["Megatron", "DeepSpeed", "collective_test", "DeepSeek", "Qwen3", "Qwen3.5"], + default="Megatron", + ) + parser.add_argument("--gpu_type", type=str, default=None), + parser.add_argument("--world_size", type=int, default=1, + help="Number of GPUs") + parser.add_argument("--tensor_model_parallel_size", type=int, default=1, + help='Degree of tensor model parallelism.') + parser.add_argument("--pipeline_model_parallel", type=int, default=1, + help='Degree of pipeline model parallelism.') + parser.add_argument('--context-parallel-size', type=int, default=1, + help='Degree of context parallelism.') + parser.add_argument("--pp_rank", type=int, default=-1, + help='Rank where encoder and decoder should be split.') + parser.add_argument("--global_batch", type=int, default=4, + help='Training batch size. If set, it should be a ' + 'multiple of micro-batch-size times data-parallel-size. ' + 'If this value is None, then ' + 'use micro-batch-size * data-parallel-size as the ' + 'global batch size. This choice will result in 1 for ' + 'number of micro-batches.') + parser.add_argument("--micro_batch", type=int, default=1, + help='Batch size per model instance (local batch size). ' + 'Global batch size is local batch size times data ' + 'parallel size times number of micro batches.' + ) + parser.add_argument("--epoch_num", type=int, default=1, + help="Number of iterations") + parser.add_argument("--computation_enable", action="store_true", help="Enable computation") + parser.add_argument("--dtype", default="bfloat16") + parser.add_argument( + "--ffn_hidden_size", + type=int, + default=None, + help="[DEPRECATED] Use --intermediate_size instead.", + ) + parser.add_argument( + "--intermediate_size", + type=int, + default=None, + help="Transformer Feed-Forward Network intermediate size. " + "For SwiGLU: the gated intermediate dim (e.g., 12288 for 8B). " + "For standard FFN: 4*hidden_size. " + "Auto-computed from --swiglu and --hidden_size if not provided.", + ) + parser.add_argument( + "--enable_visual", + action="store_true", + help="Enable visualization", + ) + parser.add_argument("--workload_only", action="store_true", help="Only generate workload") + get_model_params(parser) + get_ds_params(parser) + get_megatron_params(parser) + get_collective_test_params(parser) + get_moe_params(parser) + get_simAI_workload_params(parser) + get_aiob_params(parser) + get_deepseek_params(parser) + get_qwen3_params(parser) + args = parser.parse_args() + + assert ( + args.world_size % (args.tensor_model_parallel_size * args.pipeline_model_parallel) == 0 + ), f"world size: {args.world_size}, tp: {args.tensor_model_parallel_size}, pp: {args.pipeline_model_parallel}" + if args.moe_enable: + assert ( + args.moe_enable and args.enable_sequence_parallel + ), f"moe must be enabled with sequence parallel" + args.dp_num = args.world_size // (args.tensor_model_parallel_size * args.pipeline_model_parallel) + # assert args.global_batch % (args.dp_num * args.micro_batch) == 0, \ + # f"global_batch: {args.global_batch}, dp: {args.dp_num}, micro_batch: {args.micro_batch}" + args.num_microbatches = args.global_batch // (args.dp_num * args.micro_batch) + if args.aiob_enable and not args.computation_enable: + args.computation_enable = True + + if args.num_attention_heads is None: + args.num_attention_heads = args.num_layers + + # Aliases for HF config.json field names used by Qwen3 / Qwen3.5 models + args.num_hidden_layers = args.num_layers + if not hasattr(args, 'padded_vocab_size') or args.padded_vocab_size is None: + args.padded_vocab_size = get_padded_vocab_size(args) + # Qwen3/Qwen3.5 models use vocab_size but the embedding needs padded_vocab_size + # (divisible by tp). Alias so the HF field name picks up the padded value. + args.vocab_size = args.padded_vocab_size + + # Resolve intermediate_size: --intermediate_size > --ffn_hidden_size > auto + if args.intermediate_size is None: + args.intermediate_size = args.ffn_hidden_size + if args.intermediate_size is None: + if args.swiglu: + # SwiGLU intermediate ≈ 8/3 * hidden_size, rounded to multiple of 64 + args.intermediate_size = int((4 * args.hidden_size * 2 / 3) / 64) * 64 + else: + args.intermediate_size = 4 * args.hidden_size + # Keep ffn_hidden_size for backward compat with code still using the old name + args.ffn_hidden_size = args.intermediate_size + + if args.swiglu: + args.gated_linear_unit = True + args.bias_gelu_fusion = False + # Expert parallelism check + if args.expert_model_parallel_size > 1: + assert args.num_experts is not None, "num_experts must be non None to use expert model parallelism" + assert args.num_experts % args.expert_model_parallel_size == 0, \ + "Number of experts should be a multiple of expert model parallel_size." + assert not args.dtype == "float16", \ + "Expert parallelism is not supported with fp16 training." + if args.moe_grouped_gemm: + assert args.dtype == "bfloat16", 'Currently GroupedGEMM for MoE only supports bf16 dtype.' + if args.pipeline_model_parallel > 1 : + args.num_layers = int(args.num_layers//args.pipeline_model_parallel) + return args + + +ARGS = None + + +def get_args(): + global ARGS + if ARGS is not None: + return ARGS + ARGS = get_params() + return ARGS + + +def get_aiob_params(parser: argparse.ArgumentParser): + parser.add_argument( + "--aiob_enable", + action="store_true", + help="Enable aiob to get operation real compute time", + ) + parser.add_argument("--comp_filepath", type=str, default=None, + help="Use aiob_lib to get operation real compute time",) + parser.add_argument("--gated_linear_unit", default=False) + parser.add_argument("--bias_gelu_fusion", action="store_true", + help='Enable bias and gelu fusion.') + parser.add_argument("--openai_gelu", action="store_true", + help='Use OpenAIs GeLU implementation. This option' + 'should not be used unless for backward compatibility' + 'reasons.') + parser.add_argument("--onnx_safe", action="store_true", + help='Use workarounds for known problems with ' + 'Torch ONNX exporter') + parser.add_argument("--squared_relu", action="store_true", + help='Use squared relu activation instead of default gelu') + parser.add_argument('--recompute_activations', action='store_true', + help='recompute activation to allow for training ' + 'with larger models, sequences, and batch sizes.') + + +def get_model_params(parser: argparse.ArgumentParser): + parser.add_argument("--model_name", help="Model for training") + parser.add_argument( + "--hidden_size", type=int, help='Tansformer hidden size.', default=1024 + ) + parser.add_argument("--num_layers", type=int, help='Number of transformer layers.', default=24) + parser.add_argument( + "--seq_length", type=int, help='Maximum sequence length to process.', default=2048 + ) + parser.add_argument("--num_attention_heads", help='Number of transformer attention heads.',type=int, default=None) + parser.add_argument("--vocab_size", type=int, help='Size of vocab before EOD or padding.', default=32000) + parser.add_argument("--max_position_embeddings", type=int,help='Maximum number of position embeddings to use. ' + 'This is the size of position embedding.', default=4096) + parser.add_argument("--add_bias_linear",help='Enable bias in the linear layers', action="store_true") + parser.add_argument( + "--use_flash_attn", + action="store_true", + help="Use FlashAttention implementation of attention.", + ) + parser.add_argument( + "--swiglu", + action="store_true", + help="Use gated linear units and SiLU activation instead of default gelu", + ) + + +def get_ds_params(parser: argparse.ArgumentParser): + parser.add_argument("--stage", type=int, default=3, choices=[1, 2, 3]) + parser.add_argument("--amp_enabled", action="store_true") + parser.add_argument("--reduce_bucket_size", type=int, default=int(5e8)) + + # for stage1/2 only + parser.add_argument("--allgather_bucket_size", type=int, default=int(5e8)) + parser.add_argument("--contiguous_gradients", action="store_true") + + # for stage 3 only + parser.add_argument("--param_persistence_threshold", type=int, default=int(1e5)) + parser.add_argument( + "--model_persistence_threshold", type=int, default=int(sys.maxsize) + ) + parser.add_argument("--max_live_parameters", type=int, default=int(1e9)) + parser.add_argument("--prefetch_bucket_size", type=int, default=int(1e9)) + + +def get_megatron_params(parser: argparse.ArgumentParser): + parser.add_argument("--enable_sequence_parallel",help='Enable sequence parallel optimization.',action="store_true") + parser.add_argument( + "--use-distributed-optimizer", + action="store_true", + help="Use distributed optimizer.", + ) + parser.add_argument("--make_vocab_size_divisible_by", help='Pad the vocab size to be divisible by this value.' + 'This is added for computational efficieny reasons.',type=int, default=128) + parser.add_argument( + "--overlap_grad_reduce", + action="store_true", + default=False, + help="If set, overlap DDP grad reduce. (Not implement yet)", + ) + + +def get_collective_test_params(parser: argparse.ArgumentParser): + parser.add_argument("--begin_size", type=int, default=1048576) + parser.add_argument("--end_size", type=int, default=1048576) + parser.add_argument("--test_comm", type=str, default="all_reduce") + parser.add_argument("--iter_num", type=int, default=500) + parser.add_argument("--multi_all_reduce_enable", type=int, default=0) + + +def get_simAI_workload_params(parser: argparse.ArgumentParser): + parser.add_argument("--overlap_version", action="store_true") + +def get_moe_params(parser: argparse.ArgumentParser): + parser.add_argument('--moe_enable', action="store_true") + parser.add_argument('--expert_model_parallel_size', type=int, default=1, help='Degree of expert model parallelism.') + parser.add_argument('--num_experts', type=int, default=1, help='Number of Experts in MoE (None means no MoE)') + parser.add_argument('--moe_router_topk', type=int, default=1, help='Number of experts to route to for each token. The default is 2.') + parser.add_argument('--moe_grouped_gemm', action='store_true', + help='When there are multiple experts per rank, compress multiple local (potentially small) gemms in a single kernel launch to improve the utilization and performance by leveraging the Grouped GEMM feature introduced since CUTLASS 2.8 (https://github.com/fanshiqing/grouped_gemm).') + parser.add_argument('--activation_func', type=str,help='activation_func for mlp') + +def get_qwen3_params(parser: argparse.ArgumentParser): + parser.add_argument( + "--head_dim", + type=int, + default=128, + help="Dimension per attention head. Qwen3 uses fixed head_dim=128.", + ) + parser.add_argument( + "--num_key_value_heads", + type=int, + default=None, + help="Number of key/value heads for GQA. Defaults to num_attention_heads if not set.", + ) + + +def get_deepseek_params(parser: argparse.ArgumentParser): + parser.add_argument( + "--n_dense_layers", type=int, default=3, help="Number of dense (non-MoE) layers" + ) + parser.add_argument( + "--n_shared_expert", + type=int, + default=2, + help="Number of shared experts for DeepSeek model", + ) + parser.add_argument( + "--qk_rope_dim", + type=int, + default=64, + help="Dimention of QK with positional embeddings", + ) + parser.add_argument( + "--qk_nope_dim", + type=int, + default=128, + help="Dimention of QK without positional embeddings", + ) + parser.add_argument( + "--q_lora_rank", type=int, default=1536, help="Q down projection size" + ) + parser.add_argument( + "--kv_lora_rank", type=int, default=512, help="KV down projection size" + ) + parser.add_argument( + "--v_head_dim", type=int, default=128, help="Dimention for value projection" + ) + +def ensure_divisibility(numerator, denominator): + """Ensure that numerator is divisible by the denominator.""" + assert numerator % denominator == 0, "{} is not divisible by {}".format( + numerator, denominator + ) + + +def get_padded_vocab_size(args): + """Pad vocab size so it is divisible by model parallel size and + still having GPU friendly size.""" + + after = args.vocab_size + + multiple = args.make_vocab_size_divisible_by * args.tensor_model_parallel_size + while (after % multiple) != 0: + + after += 1 + + return after + + +def divide(numerator, denominator): + """Ensure that numerator is divisible by the denominator and return + the division value.""" + ensure_divisibility(numerator, denominator) + return numerator // denominator + +def num_parameters_to_bytes(args, params: int) -> str: + """convert parameters to MBs or GBs""" + bytes_per_param = 1 + if args.dtype == "bfloat16" or args.dtype == "float16": + bytes_per_param = 2 + else: + # default to float32, similart to AiobMegatron + bytes_per_param = 4 + b = params * bytes_per_param + gb = b / 1e9 + # if less than 1GB, print MB + if gb < 0: + return f"{b/1e6:.2f} MB" + return f"{gb:.2f} GB" + +class Strategy(str, Enum): + RoundRobin = "RoundRobin" + UniformRandom = "UniformRandom" + +def get_ep_expected_m_per_group(m, num_groups, topk, ep, strategy = "RoundRobin"): + if strategy == Strategy.RoundRobin: + return max(round(m * topk / num_groups), 1) + elif strategy == Strategy.UniformRandom: + # Simulate Global distribution to find the busiest RANK (Straggler Rank). + global_total_tokens = m * topk * ep + global_total_experts = num_groups * ep + + rng = np.random.default_rng() + + # 1. Distribute tokens to ALL global experts + all_experts_load = rng.multinomial(global_total_tokens, [1.0/global_total_experts]*global_total_experts) + + # 2. Aggregate loads per Rank + # Reshape to (ep, num_groups) and sum across experts within each rank + rank_loads = all_experts_load.reshape(ep, num_groups).sum(axis=1) + + # 3. Find the busiest Rank + max_rank_load = rank_loads.max() + + # 4. Return average load per expert on the busiest rank + return int(max_rank_load / num_groups) + else: + raise ValueError(f"Invalid strategy: {strategy}") \ No newline at end of file diff --git a/aicb/visualize/__init__.py b/aicb/visualize/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/aicb/visualize/example.html b/aicb/visualize/example.html new file mode 100644 index 000000000..26140010e --- /dev/null +++ b/aicb/visualize/example.html @@ -0,0 +1,162 @@ + + + + + Data visualization report + + + + +

Data visualization report

+ +
+

Communication type pie chart

+
+
+ +
+

Communication type scatter

+
+
+ +
+

message size CDF

+
+
+ +
+

Communication group scatter

+
+
+ +
+

Computation Communication Pattern

+
+ {% for i in range(iteration_count) %} + + {% endfor %} +
+
+
+
+ +
+

Communication Domain: Rank Connectivity Graph

+
+
+ +
+

Communication Domain: Message Size by Group

+
+
+ +
+

Ratio Pie

+
+
+ + + + \ No newline at end of file diff --git a/aicb/visualize/generate.py b/aicb/visualize/generate.py new file mode 100755 index 000000000..a9c5c7722 --- /dev/null +++ b/aicb/visualize/generate.py @@ -0,0 +1,603 @@ + +from log_analyzer.log import * +from pyecharts.charts import Pie,Scatter,Line,Graph,Bar +from pyecharts import options as opts +from typing import List, Dict +import random +import math +from collections import defaultdict +from jinja2 import Template +from pyecharts.commons.utils import JsCode +import json +import sys +import ast + +colors_group = { + "tp_group": "#376AB3", + "dp_group": "#87C0CA", + "ep_group": "#E8EDB9" , + "pp_group": "#8cc540" , + "all_reduce": "#376AB3", + "broadcast": "#87C0CA", + "all_gather": "#E8EDB9" , + "reduce_scatter": "#8cc540", + "all_to_all":"#009f5d", + "isend": "#A9A9A9" , + "irecv": "#FFD700" +} +def parse_msg_size(msg_size_str: str) -> float: + try: + result = ast.literal_eval(msg_size_str) + if isinstance(result, tuple): + return 0.0 + except (ValueError, SyntaxError): + pass + + try: + return float(msg_size_str) + except ValueError: + return 0.0 + +def custom_csv_reader(file_path: str, only_workload: bool): + with open(file_path, newline='') as csvfile: + reader = csv.reader(csvfile) + header = next(reader) + + if only_workload: + next(reader) + + for row in reader: + combined_row = [] + temp = "" + inside_tuple = False + for item in row: + if item.startswith("((") and not item.endswith("))"): + inside_tuple = True + temp += item + elif inside_tuple: + temp += "," + item + if item.endswith("))"): + inside_tuple = False + combined_row.append(temp) + temp = "" + else: + combined_row.append(item) + + yield dict(zip(header, combined_row)) + +def read_csv_and_structure_data(file_path: str, only_workload: bool) -> List[LogItem]: + log_items = [] + for row in custom_csv_reader(file_path, only_workload): + + # parse msg_size + msg_size = parse_msg_size(row['msg_size']) + + log_item = LogItem( + comm_type=CommType[row['comm_type'].split('.')[-1]], + comm_group=CommGroup[row['comm_group'].split('.')[-1]] if row['comm_group'] != 'None' else None, + comm_group_size=int(row['comm_group_size']) if row['comm_group_size'] != 'None' else None, + msg_size=msg_size, + stage=row['stage'], + dst=int(row['dst']) if row['dst'] != 'None' else None, + src=int(row['src']) if row['src'] != 'None' else None, + additional=row['additional'], + _elapsed_time=None if only_workload else float(row['_elapsed_time']), + algbw=None if only_workload else float(row['algbw']), + busbw=None if only_workload else float(row['busbw']), + count=float(row['count']) + ) + log_items.append(log_item) + + return log_items + +def split_data_by_epoch(is_comm: int,log_items: List[LogItem]) -> Dict[str, List[LogItem]]: + data_by_epoch = {'init': []} + epoch_count = 0 + current_epoch = 'init' + + for log_item in log_items: + if is_comm and log_item.comm_type == CommType.computation: + continue + + if log_item.is_epoch_end(): + data_by_epoch[current_epoch].append(log_item) + current_epoch = f'epoch_{epoch_count}' + data_by_epoch[current_epoch] = [] + epoch_count += 1 + else: + data_by_epoch[current_epoch].append(log_item) + if not data_by_epoch[current_epoch]: + del data_by_epoch[current_epoch] + return data_by_epoch +def count_by_epoch(data_by_epoch: Dict[str, List[LogItem]]) -> Dict[str, Dict[str, int]]: + comm_type_counts = {} + + for epoch, log_items in data_by_epoch.items(): + if epoch == 'init': + continue + + comm_type_counts[epoch] = {} + for log_item in log_items: + if log_item.comm_type == CommType.epoch_end: + continue + + comm_type_str = log_item.comm_type.name + if comm_type_str not in comm_type_counts[epoch]: + comm_type_counts[epoch][comm_type_str] = 0 + comm_type_counts[epoch][comm_type_str] += 1 + + return comm_type_counts +def extract_data_from_log_items(log_items: List[LogItem]) -> List: + data = [] + count_dict = defaultdict(int) + + # Count the occurrences of each (comm_type, msg_size, group) combination + for item in log_items: + if item.stage != 'init' and item.comm_type != CommType.epoch_end: + group = item.comm_group.name if item.comm_group else "unknown" + if item.msg_size > 0: + item.msg_size = item.msg_size #B + key = (item.comm_type.name, item.msg_size, group) + count_dict[key] += 1 + + # Construct the return data, including the busbw value + for item in log_items: + if item.stage != 'init' and item.comm_type != CommType.epoch_end: + group = item.comm_group.name if item.comm_group else "unknown" + if item.msg_size > 0: + key = (item.comm_type.name, item.msg_size, group) + count = count_dict[key] + data.append((item.comm_type.name, item.msg_size, group, item.busbw, count)) + return data + +def create_pie_chart_for_epoch(comm_type_counts: Dict[str, int]): + pie = Pie() + data = [(k, v) for k, v in comm_type_counts.items()] + pie.add("", data) + pie.set_global_opts( + title_opts=opts.TitleOpts(title=f"CommType Counts"), + legend_opts=opts.LegendOpts(type_="scroll", pos_left="80%", orient="vertical") + ) + pie.set_series_opts(label_opts=opts.LabelOpts(formatter="{b}: {c} ({d}%)")) + return pie + +def create_scatter_chart(Type:str,datas: List): + + scatter = Scatter() + if Type == "commtype": + index = 0 + index_y = 2 + _title="Commtype_Scatter" + elif Type == "group" : + index = 2 + index_y = 0 + _title="Commgroup_Scatter" + x_data = list(dict.fromkeys([item[index] for item in datas])) + scatter.add_xaxis(x_data) + + # colors = {group: generate_color() for group in groups} + + # Add y data to scatter with its size and color (commtype, size, group, busbw, count) + for data in datas: + y = [None] * len(x_data) + sizes = [None] * len(x_data) + + if data[index] in x_data: + y[x_data.index(data[index])] = math.log(data[1], 2) + sizes[x_data.index(data[index])] = data[1] + scatter.add_yaxis( + series_name=data[index_y], + y_axis=y, + symbol_size=30, + label_opts=opts.LabelOpts(is_show=False), + itemstyle_opts=opts.ItemStyleOpts(color=colors_group[data[index_y]]), # set group color + tooltip_opts=opts.TooltipOpts( + formatter=f"msg_size: {data[1]}
busbw: {data[3]}
count: {data[4]}" + ) + ) + + # Set global options + max_y_value =math.ceil((max(math.log(data[1], 2) for data in datas))) + min_y_value = min(math.log(data[1], 2) for data in datas) + scatter.set_global_opts( + title_opts=opts.TitleOpts(title=_title), + xaxis_opts=opts.AxisOpts( + name="Comm_type", + type_="category", + name_location="middle", + name_gap=30, + name_textstyle_opts=opts.TextStyleOpts( + font_size="15px", + font_weight='bold' + )), + yaxis_opts=opts.AxisOpts( + name="log(msg_size)", + min_=0, + max_=max_y_value, + name_location="middle", + name_gap=30, + name_textstyle_opts=opts.TextStyleOpts( + font_size="15px", + font_weight='bold' + )), + legend_opts=opts.LegendOpts( + type_="scroll", + pos_left="right", + orient="vertical", + ), + + ) + + return scatter + +def calculate_cdf_by_commtype(data: List[tuple[str, int, str, int, int]]) -> Dict[str, tuple[List[tuple[str, int, str, int, int]], np.ndarray]]: + cdf_data = defaultdict(list) + + # Split data based on comm_type + for item in data: + comm_type = item[0] + cdf_data[comm_type].append(item) + + # CDPF calculation + cdf_result = {} + for comm_type, items in cdf_data.items(): + items.sort(key=lambda x: x[1]) + msg_sizes = [math.log(item[1], 2) for item in items] + cdf = np.arange(1, len(msg_sizes) + 1) / len(msg_sizes) + cdf_result[comm_type] = (items, cdf) + + return cdf_result +def create_cdf_chart_by_commtype(cdf_data: Dict[str, tuple[List[tuple[str, int, str, int, int]], np.ndarray]]) -> Line: + line = Line() + + + # Generate a CDF line for each comm_type + for comm_type, (data, cdf) in cdf_data.items(): + + x_data = [math.log(item[1], 2) for item in data] + y_data = cdf.tolist() + msg_sizes = [item[1] for item in data] + random_color = "#{:06x}".format(random.randint(0, 0xFFFFFF)) + + line.add_xaxis(x_data) + line.add_yaxis( + series_name=comm_type, + y_axis=y_data, + label_opts=opts.LabelOpts(is_show=False), + tooltip_opts=opts.TooltipOpts( + formatter=JsCode( + """ + function(params) { + var msgSizes = %s; + return params.seriesName + '
' + + 'msg_size: ' + msgSizes[params.dataIndex].toFixed(2) + '
' + + 'CDF: ' + params.value[1].toFixed(4); + } + """ % str(msg_sizes) + ), + ), + ) + + line.set_global_opts( + title_opts=opts.TitleOpts(title="msg_size CDF"), + xaxis_opts=opts.AxisOpts( + type_="value", + name="log(msg_size)", + name_location="middle", + name_gap=30, + name_textstyle_opts=opts.TextStyleOpts(font_size=14, font_weight='bold') + ), + yaxis_opts=opts.AxisOpts( + type_="value", + name="CDF", + name_location="middle", + name_gap=30, + name_textstyle_opts=opts.TextStyleOpts(font_size=14, font_weight='bold') + ), + tooltip_opts=opts.TooltipOpts(trigger="axis"), + legend_opts=opts.LegendOpts( + type_="scroll", + pos_left="right", + orient="vertical", + ), + + ) + + return line + + + +def create_timeline_chart(epoch_data): + def process_items(items): + timeline_comp = [] + timeline_comm = [] + x_data = [] + current_time = 0 + len_data = len(items) + 1 + x_data.append(f"{current_time:.3f}") + for i,item in enumerate(items): + if item.comm_type != CommType.epoch_end: + # y = [None] * len_data + start_time = current_time + end_time = current_time + item._elapsed_time + if i < len_data - 1 and item.comm_type == CommType.computation: + y = [None] * (i + 2) + y[i] = 1 + y[i+1] = 1 + timeline_comp.append({ + 'value': y, + 'stage': item.stage, + 'elapsed_time': item._elapsed_time, + 'comm_type':item.comm_type + }) + elif i < len_data - 1: + y = [None] * (i + 2) + y[i] = 2 + y[i+1] = 2 + timeline_comm.append({ + 'value': y, + 'stage': item.stage, + 'elapsed_time': item._elapsed_time, + 'comm_type':item.comm_type + }) + x_data.append(f"{end_time:.3f}") + current_time = end_time + return timeline_comp,timeline_comm,x_data + + computation_timeline = [] + communication_timeline = [] + computation_timeline,communication_timeline,x_data = process_items(epoch_data) + # Calculate computation time and communication time + total_computation_time = sum(item['elapsed_time'] for item in computation_timeline) + total_communication_time = sum(item['elapsed_time'] for item in communication_timeline) + + line = Line() + line.add_xaxis(x_data) + + for comp_y in computation_timeline: + line.add_yaxis( + "Computation", + comp_y['value'], + is_connect_nones=False, + label_opts=opts.LabelOpts(is_show=False), + linestyle_opts=opts.LineStyleOpts(width=5), + tooltip_opts=opts.TooltipOpts( + formatter=f"stage: {comp_y['stage']}
elapsed_time: {comp_y['elapsed_time']}
comm_type: {comp_y['comm_type']}" + ), + ) + + + for comm_y in communication_timeline: + + line.add_yaxis( + "communication", + comm_y['value'], + is_connect_nones=False, + label_opts=opts.LabelOpts(is_show=False), + linestyle_opts=opts.LineStyleOpts(width=5), + tooltip_opts=opts.TooltipOpts( + formatter=f"stage: {comm_y['stage']}
elapsed_time: {comm_y['elapsed_time']}
comm_type: {comm_y['comm_type']}" + ), + ) + + + line.set_global_opts( + title_opts=opts.TitleOpts(title="Computation and Communication Timeline"), + xaxis_opts=opts.AxisOpts(name="Time (ms)",type_ = "value"),#type_ = "value" + yaxis_opts=opts.AxisOpts(name="Type", max_=3,axislabel_opts=opts.LabelOpts(is_show=False),axistick_opts=opts.AxisTickOpts(is_show=False) + ), + tooltip_opts=opts.TooltipOpts(trigger="axis"), + + datazoom_opts=[ + opts.DataZoomOpts(type_="slider", range_start=0, range_end=100), + opts.DataZoomOpts(type_="inside", range_start=0, range_end=100), + ], + legend_opts=opts.LegendOpts( + type_="scroll", + pos_left="right", + orient="vertical", + ), + + ) + #comp—comm Pie + pie = Pie() + pie.add( + "", + [ + ("Computation", total_computation_time), + ("Communication", total_communication_time) + ], + radius=["40%", "75%"], + ) + pie.set_global_opts( + title_opts=opts.TitleOpts(title="Computation vs Communication time Ratio"), + legend_opts=opts.LegendOpts(type_="scroll", pos_left="80%", orient="vertical") + ) + pie.set_series_opts(label_opts=opts.LabelOpts(formatter="{b}:{d}%")) + return line,pie +def extract_iteration(epoch_data): + data_by_iter = [] + current_iter = [] + broadcast_count = 0 + + for item in epoch_data: + if item.comm_type == CommType.epoch_end: + if current_iter: + data_by_iter.append(current_iter) + break + + if broadcast_count < 2: + if item.comm_type == CommType.broadcast: + broadcast_count += 1 + current_iter.append(item) + else: + if item.comm_type == CommType.broadcast: + data_by_iter.append(current_iter) + current_iter = [item] + broadcast_count = 1 + else: + current_iter.append(item) + + + + return data_by_iter + +def create_ratio_pie(epoch_data): + total_computation_time = total_communication_time = 0 + for item in epoch_data: + if item.comm_type == CommType.computation: + total_computation_time += item._elapsed_time + elif item.comm_type != CommType.epoch_end: + total_communication_time += item._elapsed_time + + total_ratio_pie = Pie() + total_ratio_pie.add( + "", + [ + ("Computation", total_computation_time), + ("Communication", total_communication_time) + ], + radius=["40%", "75%"], + ) + total_ratio_pie.set_global_opts( + title_opts=opts.TitleOpts(title="Overall Computation vs Communication time Ratio"), + legend_opts=opts.LegendOpts(type_="scroll", pos_left="80%", orient="vertical") + ) + total_ratio_pie.set_series_opts(label_opts=opts.LabelOpts(formatter="{b}: {d}%")) + return total_ratio_pie + +def extract_ranks_data(raw_rows): + domain_edges = defaultdict(lambda: defaultdict(int)) + domain_msg_total = defaultdict(int) + domain_msg_count = defaultdict(int) + for row in raw_rows: + cg_raw = row.get('comm_group', 'None') + if cg_raw == 'None' or cg_raw == '': + continue + group_name = cg_raw.split('.')[-1] if '.' in cg_raw else cg_raw + msg_size = parse_msg_size(row.get('msg_size', '0')) + ranks_str = row.get('ranks', '') + if ranks_str: + ranks = [int(r) for r in ranks_str.split(',') if r.strip()] + for i in range(len(ranks)): + for j in range(i + 1, len(ranks)): + edge = (min(ranks[i], ranks[j]), max(ranks[i], ranks[j])) + domain_edges[group_name][edge] += 1 + domain_msg_total[group_name] += msg_size + domain_msg_count[group_name] += 1 + return {'edges': domain_edges, 'msg_total': domain_msg_total, 'msg_count': domain_msg_count} + + +def create_domain_flow_graph(domain_ranks): + graph = Graph(init_opts=opts.InitOpts(width="100%", height="500px")) + all_ranks = set() + categories = [] + links = [] + group_colors = {'tp_group': '#376AB3', 'dp_group': '#87C0CA', 'pp_group': '#8cc540', 'ep_group': '#E8EDB9', 'ep_dp_group': '#f4a261', 'ep_tp_group': '#e76f51'} + for group_name, edges in domain_ranks['edges'].items(): + color = group_colors.get(group_name, '#999999') + if group_name not in [c['name'] for c in categories]: + categories.append({'name': group_name, 'itemStyle': {'color': color}}) + for (src, dst), count in edges.items(): + all_ranks.add(src); all_ranks.add(dst) + links.append({'source': str(src), 'target': str(dst), 'value': count, 'lineStyle': {'color': color, 'curveness': 0.15, 'opacity': 0.6}}) + nodes = [{'name': str(r), 'symbolSize': 20, 'category': 0} for r in sorted(all_ranks)] + graph.add("", nodes, links, categories, repulsion=200, edge_length=150, is_draggable=True, label_opts=opts.LabelOpts(is_show=True, position="inside")) + graph.set_global_opts(title_opts=opts.TitleOpts(title="Communication Domain: Rank Connectivity Graph"), legend_opts=opts.LegendOpts(type_="scroll", pos_left="80%", orient="vertical")) + return graph + + +def create_domain_msg_size_bar(domain_ranks): + bar = Bar(init_opts=opts.InitOpts(width="100%", height="400px")) + groups = list(domain_ranks['msg_total'].keys()) + totals = [domain_ranks['msg_total'][g] for g in groups] + bar.add_xaxis(groups) + bar.add_yaxis("Total Bytes", totals, label_opts=opts.LabelOpts(formatter="{c}"), itemstyle_opts=opts.ItemStyleOpts(color='#376AB3')) + bar.set_global_opts(title_opts=opts.TitleOpts(title="Communication Domain: Total Message Size by Group"), xaxis_opts=opts.AxisOpts(name="Comm Group", axislabel_opts=opts.LabelOpts(rotate=30)), yaxis_opts=opts.AxisOpts(name="Total Bytes")) + return bar + + +def visualize_output(filepath,only_workload:bool): + + log_items = read_csv_and_structure_data(filepath,only_workload) + raw_rows = list(custom_csv_reader(filepath, only_workload)) + + #pie + data_by_epoch_comm = split_data_by_epoch(1,log_items) #only comm + comm_type_counts = count_by_epoch(data_by_epoch_comm) + pie_chart = create_pie_chart_for_epoch(comm_type_counts['epoch_0']) + + + data = extract_data_from_log_items(data_by_epoch_comm['epoch_0']) + #commtype Scatter + effect_scatter_by_commtype = create_scatter_chart("commtype",data) + + #commtype cdf + cdf_data = calculate_cdf_by_commtype(data) + cdf_chart = create_cdf_chart_by_commtype(cdf_data) + + + #group Scatter + effect_scatter_by_group = create_scatter_chart("group",data) + + #ranks-based domain charts + domain_ranks = extract_ranks_data(raw_rows) + domain_graph = create_domain_flow_graph(domain_ranks) + domain_bar = create_domain_msg_size_bar(domain_ranks) + + #comp-comm pattern + data_by_epoch = split_data_by_epoch(0, log_items) + + timeline_charts = [] + ratio_pies = [] + all_ratio_pie = [] + if not only_workload: + all_iterations = extract_iteration(data_by_epoch['epoch_0']) + for iteration in all_iterations: + timeline_chart, ratio_pie = create_timeline_chart(iteration) + timeline_charts.append(timeline_chart.dump_options()) + ratio_pies.append(ratio_pie.dump_options()) + all_ratio_pie = create_ratio_pie(data_by_epoch['epoch_0']) + + else: + all_ratio_pie = None + + context = { + 'pie_chart_js': pie_chart.dump_options(), + 'scatter_by_commtype_js': effect_scatter_by_commtype.dump_options(), + 'cdf_chart_js': cdf_chart.dump_options(), + 'scatter_by_group_js': effect_scatter_by_group.dump_options(), + 'timeline_charts_js': json.dumps(timeline_charts), + 'ratio_pies_js': json.dumps(ratio_pies), + 'iteration_count': len(all_iterations) if not only_workload else 0, + 'all_ratio_pie':all_ratio_pie.dump_options() if all_ratio_pie else None, + 'domain_graph_js': domain_graph.dump_options(), + 'domain_bar_js': domain_bar.dump_options(), + } + + # read Example.html + with open('visualize/example.html', 'r', encoding='utf-8') as f: + template = Template(f.read()) + + + rendered_html = template.render(**context) + # write to file + default_folder_path = 'results/visual_output' + if not os.path.exists(default_folder_path): + os.makedirs(default_folder_path, exist_ok=True) + filename = os.path.basename(filepath).split(".")[0]+'.html' + output_file = os.path.join('results/visual_output',filename) + with open(output_file, 'w', encoding='utf-8') as f: + f.write(rendered_html) + + print(f"Report generated:{output_file}") + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage: script.py [only_workload]") + sys.exit(1) + + filepath = sys.argv[1] + flag = (len(sys.argv) > 2 and sys.argv[2] == 'only_workload') + print(f'only workload flag is {flag}') + visualize_output(filepath,flag) \ No newline at end of file diff --git a/aicb/visualize/inputs/A100_example.csv b/aicb/visualize/inputs/A100_example.csv new file mode 100644 index 000000000..aa5f13f88 --- /dev/null +++ b/aicb/visualize/inputs/A100_example.csv @@ -0,0 +1,8539 @@ +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,4,8,init.model_setup,None,None,,3380.222900390625,0.0,0.0,1 +CommType.all_reduce,CommGroup.dp_group,4,8,init.model_setup,None,None,,0.15667200088500977,0.0,0.0,1 +CommType.all_reduce,CommGroup.dp_group,4,8,init.model_setup,None,None,,0.09625600278377533,0.0,0.0,1 +CommType.all_reduce,CommGroup.dp_group,4,8,init.model_setup,None,None,,0.0870399996638298,0.0,0.0,1 +CommType.all_gather,CommGroup.dp_group,4,32,init.model_setup,None,None,,0.11776000261306763,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,24,init.model_setup,None,0,,675.4068603515625,0.0,0.0,1 +CommType.all_gather,CommGroup.dp_group,4,64,init.model_setup,None,None,,0.15769599378108978,0.0,0.0,1 +CommType.epoch_end,None,None,0,,None,None,,7950.41298866272,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,0.131071999669075,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,16448,forward_step,None,0,,0.09830400347709656,0.17,0.17,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronEmbedding,None,None,,0.20070399343967438,83.59,146.29,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.16281600296497345,103.04,90.16,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.09011200070381165,0.73,0.64,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,327.4864501953125,0.03,0.03,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,2406.90576171875,0.01,0.01,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.6010879874229431,223.29,195.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5386239886283875,249.19,218.04,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2467840015888214,135.97,135.97,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12390399724245071,67.7,67.7,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.15564799308776855,107.79,94.32,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08396799862384796,0.78,0.68,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.52019202709198,258.02,225.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.09113600105047226,0.72,0.63,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22220799326896667,151.0,151.0,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08396799862384796,0.78,0.68,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.14643199741840363,57.29,57.29,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22220799326896667,151.0,151.0,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08191999793052673,0.8,0.7,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12492799758911133,67.15,67.15,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07680000364780426,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12697599828243256,66.06,66.06,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08294399827718735,0.79,0.69,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5273600220680237,254.51,222.7,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5181440114974976,259.04,226.66,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11980800330638885,70.02,70.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08908800035715103,0.74,0.64,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08499199897050858,0.77,0.67,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07680000364780426,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08294399827718735,0.79,0.69,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08191999793052673,0.8,0.7,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.24268800020217896,138.26,138.26,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1669120043516159,100.52,87.95,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08191999793052673,0.8,0.7,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07680000364780426,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12800000607967377,65.54,65.54,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12083200365304947,69.42,69.42,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08294399827718735,0.79,0.69,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08191999793052673,0.8,0.7,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0870399996638298,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2303999960422516,145.64,145.64,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14950400590896606,112.22,98.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08294399827718735,0.79,0.69,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12185599654912949,68.84,68.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1372160017490387,61.13,61.13,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14950400590896606,112.22,98.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08191999793052673,0.8,0.7,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.24371199309825897,137.68,137.68,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5191680192947388,258.52,226.21,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07680000364780426,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12083200365304947,69.42,69.42,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.23244799673557281,144.35,144.35,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5212159752845764,257.51,225.32,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08908800035715103,0.74,0.64,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08396799862384796,0.78,0.68,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.13516800105571747,62.06,62.06,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5191680192947388,258.52,226.21,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2242559939622879,149.63,149.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07680000364780426,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22937600314617157,146.29,146.29,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08191999793052673,0.8,0.7,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07680000364780426,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.52019202709198,258.02,225.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5375999808311462,249.66,218.45,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.5355520248413086,62.65,62.65,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07680000364780426,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.131071999669075,64.0,64.0,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08294399827718735,0.79,0.69,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.26521599292755127,126.52,126.52,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12595200538635254,66.6,66.6,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08191999793052673,0.8,0.7,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.08908800035715103,0.09,0.16,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.0798719972372055,0.1,0.18,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07884799689054489,0.1,0.18,1 +CommType.all_reduce,CommGroup.dp_group,4,4,forward_step.average_losses_across_data_parallel_group,None,None,,0.08191999793052673,0.0,0.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.17100800573825836,98.11,85.84,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12697599828243256,132.13,115.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15360000729560852,109.23,95.57,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.16486400365829468,101.76,89.04,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.158720001578331,105.7,92.49,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.15462400019168854,108.5,94.94,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronEmbedding,None,None,,0.2017280012369156,83.17,145.54,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,0.08908800035715103,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,16448,forward_step,None,0,,0.07680000364780426,0.21,0.21,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronEmbedding,None,None,,0.1812479943037033,92.56,161.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1454080045223236,57.69,57.69,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.24371199309825897,137.68,137.68,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08089599758386612,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07680000364780426,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.15769599378108978,106.39,93.09,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2017280012369156,166.34,166.34,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08191999793052673,0.8,0.7,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5017600059509277,267.49,234.06,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08499199897050858,0.77,0.67,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5017600059509277,267.49,234.06,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12595200538635254,66.6,66.6,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22220799326896667,151.0,151.0,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.6410239934921265,209.38,183.21,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5611519813537598,239.18,209.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.289792001247406,115.79,115.79,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.13209599256515503,63.5,63.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.16281600296497345,103.04,90.16,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08294399827718735,0.79,0.69,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07680000364780426,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22937600314617157,146.29,146.29,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5375999808311462,249.66,218.45,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20070399343967438,167.18,167.18,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08396799862384796,0.78,0.68,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.23449599742889404,143.09,143.09,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5212159752845764,257.51,225.32,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.52019202709198,258.02,225.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07168000191450119,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,59.507713317871094,0.28,0.25,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1730560064315796,96.95,84.83,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08499199897050858,0.77,0.67,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07680000364780426,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.13516800105571747,62.06,62.06,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,13.386752128601074,2.51,2.51,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5314559936523438,252.55,220.98,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5212159752845764,257.51,225.32,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22220799326896667,151.0,151.0,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.52019202709198,258.02,225.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22118400037288666,151.7,151.7,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12697599828243256,66.06,66.06,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08089599758386612,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.15769599378108978,106.39,93.09,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07680000364780426,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1454080045223236,57.69,57.69,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2242559939622879,149.63,149.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08294399827718735,0.79,0.69,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08396799862384796,0.78,0.68,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10342399775981903,81.11,81.11,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11980800330638885,70.02,70.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08499199897050858,0.77,0.67,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.08908800035715103,0.09,0.16,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.0798719972372055,0.1,0.18,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07680000364780426,0.11,0.19,1 +CommType.all_reduce,CommGroup.dp_group,4,4,forward_step.average_losses_across_data_parallel_group,None,None,,0.08294399827718735,0.0,0.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.17715199291706085,94.71,82.87,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.15462400019168854,108.5,94.94,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.159743994474411,105.03,91.9,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronEmbedding,None,None,,0.1884160041809082,89.04,155.83,1 +CommType.reduce_scatter,CommGroup.dp_group,4,35634282496,step,None,None,,129.74490356445312,274.65,240.32,1 +CommType.all_gather,CommGroup.dp_group,4,17817141248,step,None,None,,63.13471984863281,282.21,246.93,1 +CommType.all_reduce,CommGroup.tp_group,2,0,step._allreduce_layernorm_grads,None,None,,0.14028799533843994,0.0,0.0,1 +CommType.all_reduce,CommGroup.tp_group,2,4,step.check_for_nan,None,None,,0.09011200070381165,0.0,0.0,1 +CommType.epoch_end,None,None,0,,None,None,,3298.4800338745117,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,0.09625600278377533,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,16448,forward_step,None,0,,0.08191999793052673,0.2,0.2,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronEmbedding,None,None,,0.1884160041809082,89.04,155.83,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.13209599256515503,63.5,63.5,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2303999960422516,145.64,145.64,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5273600220680237,254.51,222.7,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12697599828243256,66.06,66.06,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.23244799673557281,144.35,144.35,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12800000607967377,65.54,65.54,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.16076800227165222,104.36,91.31,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.27033600211143494,124.12,124.12,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11980800330638885,70.02,70.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08191999793052673,0.8,0.7,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5253120064735413,255.5,223.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0870399996638298,0.75,0.66,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.25600001215934753,131.07,131.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11980800330638885,70.02,70.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2242559939622879,149.63,149.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2303999960422516,145.64,145.64,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22015999257564545,152.41,152.41,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.20684799551963806,81.11,70.97,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.09011200070381165,0.09,0.16,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.08089599758386612,0.1,0.18,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07680000364780426,0.11,0.19,1 +CommType.all_reduce,CommGroup.dp_group,4,4,forward_step.average_losses_across_data_parallel_group,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.16998399794101715,98.7,86.36,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14950400590896606,112.22,98.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronEmbedding,None,None,,0.1884160041809082,89.04,155.83,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,0.0870399996638298,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,16448,forward_step,None,0,,0.07680000364780426,0.21,0.21,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronEmbedding,None,None,,0.17817600071430206,94.16,164.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.15564799308776855,107.79,94.32,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1525759994983673,54.98,54.98,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.26521599292755127,126.52,126.52,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11980800330638885,70.02,70.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22118400037288666,151.7,151.7,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2017280012369156,166.34,166.34,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08601599931716919,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21913599967956543,153.12,153.12,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12083200365304947,69.42,69.42,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5314559936523438,252.55,220.98,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22015999257564545,152.41,152.41,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12083200365304947,69.42,69.42,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.15360000729560852,109.23,95.57,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12083200365304947,69.42,69.42,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22015999257564545,152.41,152.41,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08601599931716919,0.76,0.67,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.158720001578331,105.7,92.49,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0870399996638298,0.75,0.66,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.13312000036239624,63.02,63.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22118400037288666,151.7,151.7,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.16179199516773224,103.7,90.73,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22118400037288666,151.7,151.7,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22015999257564545,152.41,152.41,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22015999257564545,152.41,152.41,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22015999257564545,152.41,152.41,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5191680192947388,258.52,226.21,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22220799326896667,151.0,151.0,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07168000191450119,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14950400590896606,112.22,98.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.24473600089550018,137.1,137.1,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2303999960422516,145.64,145.64,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22835199534893036,146.94,146.94,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10342399775981903,81.11,81.11,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12083200365304947,69.42,69.42,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22323200106620789,150.31,150.31,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2242559939622879,149.63,149.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2252800017595291,148.95,148.95,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22220799326896667,151.0,151.0,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14950400590896606,112.22,98.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.08908800035715103,0.09,0.16,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07782399654388428,0.11,0.18,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07577600330114365,0.11,0.19,1 +CommType.all_reduce,CommGroup.dp_group,4,4,forward_step.average_losses_across_data_parallel_group,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15462400019168854,108.5,94.94,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1515520066022873,110.7,96.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12697599828243256,132.13,115.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1730560064315796,96.95,84.83,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronEmbedding,None,None,,0.18636800348758698,90.02,157.54,1 +CommType.reduce_scatter,CommGroup.dp_group,4,35634282496,step,None,None,,128.3962860107422,277.53,242.84,1 +CommType.all_gather,CommGroup.dp_group,4,17817141248,step,None,None,,63.15827178955078,282.1,246.84,1 +CommType.all_reduce,CommGroup.tp_group,2,0,step._allreduce_layernorm_grads,None,None,,0.11980800330638885,0.0,0.0,1 +CommType.all_reduce,CommGroup.tp_group,2,4,step.check_for_nan,None,None,,0.08089599758386612,0.0,0.0,1 +CommType.epoch_end,None,None,0,,None,None,,433.2258701324463,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,0.09625600278377533,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,16448,forward_step,None,0,,0.08089599758386612,0.2,0.2,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronEmbedding,None,None,,0.18636800348758698,90.02,157.54,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.14028799533843994,59.8,59.8,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22323200106620789,150.31,150.31,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08294399827718735,0.79,0.69,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08191999793052673,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2017280012369156,166.34,166.34,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12595200538635254,66.6,66.6,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2467840015888214,135.97,135.97,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08499199897050858,0.77,0.67,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22630399465560913,148.27,148.27,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11980800330638885,70.02,70.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08601599931716919,0.76,0.67,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5191680192947388,258.52,226.21,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2017280012369156,166.34,166.34,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21913599967956543,153.12,153.12,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22220799326896667,151.0,151.0,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5222399830818176,257.0,224.88,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5181440114974976,259.04,226.66,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.16179199516773224,51.85,51.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07168000191450119,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.23244799673557281,144.35,144.35,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5242879986763,256.0,224.0,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.18943999707698822,88.56,77.49,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12492799758911133,67.15,67.15,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2242559939622879,149.63,149.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22015999257564545,152.41,152.41,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,4.780032157897949,7.02,7.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.16998399794101715,49.35,49.35,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.15360000729560852,109.23,95.57,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.16076800227165222,104.36,91.31,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.1157120019197464,0.57,0.5,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.10956799983978271,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.159743994474411,52.51,52.51,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,3.2430078983306885,10.35,10.35,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0942080020904541,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08089599758386612,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.09011200070381165,0.09,0.16,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07577600330114365,0.11,0.19,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07372800260782242,0.11,0.19,1 +CommType.all_reduce,CommGroup.dp_group,4,4,forward_step.average_losses_across_data_parallel_group,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15564799308776855,107.79,94.32,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.20787200331687927,80.71,70.62,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15564799308776855,107.79,94.32,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15769599378108978,106.39,93.09,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronEmbedding,None,None,,0.1884160041809082,89.04,155.83,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,0.08601599931716919,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,16448,forward_step,None,0,,0.0798719972372055,0.21,0.21,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronEmbedding,None,None,,0.17817600071430206,94.16,164.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14950400590896606,112.22,98.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08191999793052673,0.8,0.7,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.131071999669075,64.0,64.0,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2549760043621063,131.6,131.6,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.14950400590896606,56.11,56.11,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11980800330638885,70.02,70.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22630399465560913,148.27,148.27,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08089599758386612,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5283839702606201,254.02,222.26,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5017600059509277,267.49,234.06,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2017280012369156,166.34,166.34,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10342399775981903,81.11,81.11,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.131071999669075,64.0,64.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1515520066022873,110.7,96.86,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.10035199671983719,0.65,0.57,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.10239999741315842,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2734079957008362,122.73,122.73,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,3.6730880737304688,36.54,31.97,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5212159752845764,257.51,225.32,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.23654399812221527,141.85,141.85,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12390399724245071,67.7,67.7,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1525759994983673,109.96,96.21,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.14950400590896606,56.11,56.11,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,3.625983953475952,9.25,9.25,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5191680192947388,258.52,226.21,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2314240038394928,144.99,144.99,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,6.06822395324707,5.53,5.53,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08499199897050858,0.77,0.67,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0870399996638298,0.75,0.66,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5017600059509277,267.49,234.06,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22630399465560913,148.27,148.27,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08396799862384796,0.78,0.68,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11980800330638885,70.02,70.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5181440114974976,259.04,226.66,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.09932799637317657,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22732800245285034,147.6,147.6,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5283839702606201,254.02,222.26,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.23449599742889404,143.09,143.09,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12697599828243256,66.06,66.06,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,6.961152076721191,1.21,1.21,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.09523200243711472,0.09,0.15,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07884799689054489,0.1,0.18,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07577600330114365,0.11,0.19,1 +CommType.all_reduce,CommGroup.dp_group,4,4,forward_step.average_losses_across_data_parallel_group,None,None,,0.08089599758386612,0.0,0.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.33689600229263306,49.8,43.57,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1515520066022873,110.7,96.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.16793599724769592,99.9,87.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1955839991569519,85.78,75.06,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.16793599724769592,99.9,87.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12697599828243256,132.13,115.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1515520066022873,110.7,96.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15667200088500977,107.08,93.7,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronEmbedding,None,None,,0.187391996383667,89.53,156.68,1 +CommType.reduce_scatter,CommGroup.dp_group,4,35634282496,step,None,None,,128.92466735839844,276.4,241.85,1 +CommType.all_gather,CommGroup.dp_group,4,17817141248,step,None,None,,63.47468948364258,280.7,245.61,1 +CommType.all_reduce,CommGroup.tp_group,2,0,step._allreduce_layernorm_grads,None,None,,0.13209599256515503,0.0,0.0,1 +CommType.all_reduce,CommGroup.tp_group,2,4,step.check_for_nan,None,None,,0.08396799862384796,0.0,0.0,1 +CommType.epoch_end,None,None,0,,None,None,,462.3074531555176,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,0.09318400174379349,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,16448,forward_step,None,0,,0.08499199897050858,0.19,0.19,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronEmbedding,None,None,,0.18636800348758698,90.02,157.54,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08601599931716919,0.76,0.67,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.13516800105571747,62.06,62.06,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22118400037288666,151.7,151.7,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2017280012369156,166.34,166.34,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2017280012369156,166.34,166.34,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08191999793052673,0.8,0.7,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.14847999811172485,56.5,56.5,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.23449599742889404,143.09,143.09,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22118400037288666,151.7,151.7,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22118400037288666,151.7,151.7,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22630399465560913,148.27,148.27,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1382399946451187,60.68,60.68,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1515520066022873,110.7,96.86,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.23347200453281403,143.72,143.72,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5273600220680237,254.51,222.7,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22323200106620789,150.31,150.31,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22118400037288666,151.7,151.7,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5191680192947388,258.52,226.21,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22118400037288666,151.7,151.7,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07065600156784058,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22015999257564545,152.41,152.41,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5232639908790588,256.5,224.44,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12595200538635254,66.6,66.6,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.44441598653793335,75.5,75.5,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12185599654912949,68.84,68.84,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08396799862384796,0.78,0.68,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12800000607967377,65.54,65.54,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14950400590896606,112.22,98.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08396799862384796,0.78,0.68,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08396799862384796,0.78,0.68,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08294399827718735,0.79,0.69,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.08806400001049042,0.09,0.16,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07782399654388428,0.11,0.18,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07372800260782242,0.11,0.19,1 +CommType.all_reduce,CommGroup.dp_group,4,4,forward_step.average_losses_across_data_parallel_group,None,None,,0.08089599758386612,0.0,0.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12697599828243256,132.13,115.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14950400590896606,112.22,98.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15462400019168854,108.5,94.94,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronEmbedding,None,None,,0.18534399569034576,90.52,158.41,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,0.10035199671983719,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,16448,forward_step,None,0,,0.08396799862384796,0.2,0.2,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronEmbedding,None,None,,0.1802240014076233,93.09,162.91,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14950400590896606,112.22,98.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.14028799533843994,59.8,59.8,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.25702399015426636,130.55,130.55,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12083200365304947,69.42,69.42,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2017280012369156,166.34,166.34,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5222399830818176,257.0,224.88,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5191680192947388,258.52,226.21,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12492799758911133,67.15,67.15,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.15769599378108978,106.39,93.09,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14950400590896606,112.22,98.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5181440114974976,259.04,226.66,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07168000191450119,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12083200365304947,69.42,69.42,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22732800245285034,147.6,147.6,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1515520066022873,110.7,96.86,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.24063999950885773,139.44,139.44,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22015999257564545,152.41,152.41,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07168000191450119,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5212159752845764,257.51,225.32,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.16179199516773224,103.7,90.73,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08396799862384796,0.78,0.68,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07475200295448303,0.88,0.77,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.130048006772995,64.5,64.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22835199534893036,146.94,146.94,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2017280012369156,166.34,166.34,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07680000364780426,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11980800330638885,70.02,70.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08294399827718735,0.79,0.69,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22015999257564545,152.41,152.41,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.15564799308776855,107.79,94.32,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22220799326896667,151.0,151.0,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07168000191450119,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.08806400001049042,0.09,0.16,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07782399654388428,0.11,0.18,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07782399654388428,0.11,0.18,1 +CommType.all_reduce,CommGroup.dp_group,4,4,forward_step.average_losses_across_data_parallel_group,None,None,,0.08089599758386612,0.0,0.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15360000729560852,109.23,95.57,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14950400590896606,112.22,98.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1669120043516159,100.52,87.95,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronEmbedding,None,None,,0.21606400609016418,77.65,135.89,1 +CommType.reduce_scatter,CommGroup.dp_group,4,35634282496,step,None,None,,128.9574432373047,276.33,241.79,1 +CommType.all_gather,CommGroup.dp_group,4,17817141248,step,None,None,,63.303680419921875,281.46,246.27,1 +CommType.all_reduce,CommGroup.tp_group,2,0,step._allreduce_layernorm_grads,None,None,,0.15564799308776855,0.0,0.0,1 +CommType.all_reduce,CommGroup.tp_group,2,4,step.check_for_nan,None,None,,0.08191999793052673,0.0,0.0,1 +CommType.epoch_end,None,None,0,,None,None,,433.69078636169434,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,0.09625600278377533,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,16448,forward_step,None,0,,0.08191999793052673,0.2,0.2,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronEmbedding,None,None,,0.19046400487422943,88.09,154.15,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.13414399325847626,62.53,62.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22015999257564545,152.41,152.41,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1228799968957901,68.27,68.27,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.52019202709198,258.02,225.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08089599758386612,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.23347200453281403,143.72,143.72,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07680000364780426,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.24166400730609894,138.85,138.85,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.13312000036239624,63.02,63.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1669120043516159,100.52,87.95,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08499199897050858,0.77,0.67,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5212159752845764,257.51,225.32,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11980800330638885,70.02,70.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0870399996638298,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2017280012369156,166.34,166.34,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07065600156784058,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07168000191450119,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2017280012369156,166.34,166.34,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.13209599256515503,63.5,63.5,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21913599967956543,153.12,153.12,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12595200538635254,66.6,66.6,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12800000607967377,65.54,65.54,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12185599654912949,68.84,68.84,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.131071999669075,64.0,64.0,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.4997119903564453,268.59,235.02,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.19968000054359436,168.04,168.04,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07168000191450119,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12697599828243256,66.06,66.06,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.23449599742889404,143.09,143.09,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.11673600226640701,0.56,0.49,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12083200365304947,69.42,69.42,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07168000191450119,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.23347200453281403,143.72,143.72,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22118400037288666,151.7,151.7,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.14745600521564484,56.89,56.89,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07168000191450119,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22118400037288666,151.7,151.7,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07168000191450119,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.08806400001049042,0.09,0.16,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.08499199897050858,0.1,0.17,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07577600330114365,0.11,0.19,1 +CommType.all_reduce,CommGroup.dp_group,4,4,forward_step.average_losses_across_data_parallel_group,None,None,,0.08191999793052673,0.0,0.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.17203199863433838,97.52,85.33,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.16793599724769592,99.9,87.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1515520066022873,110.7,96.86,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12697599828243256,132.13,115.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14950400590896606,112.22,98.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.15360000729560852,109.23,95.57,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12697599828243256,132.13,115.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.15667200088500977,107.08,93.7,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14950400590896606,112.22,98.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronEmbedding,None,None,,0.1955839991569519,85.78,150.12,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,0.0870399996638298,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,16448,forward_step,None,0,,0.0798719972372055,0.21,0.21,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronEmbedding,None,None,,0.18636800348758698,90.02,157.54,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.158720001578331,105.7,92.49,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.13209599256515503,63.5,63.5,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.25600001215934753,131.07,131.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12083200365304947,69.42,69.42,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1228799968957901,68.27,68.27,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1525759994983673,109.96,96.21,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5017600059509277,267.49,234.06,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5222399830818176,257.0,224.88,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1515520066022873,110.7,96.86,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12492799758911133,67.15,67.15,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22732800245285034,147.6,147.6,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10342399775981903,81.11,81.11,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10342399775981903,81.11,81.11,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2017280012369156,166.34,166.34,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0870399996638298,0.75,0.66,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5242879986763,256.0,224.0,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21913599967956543,153.12,153.12,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5232639908790588,256.5,224.44,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2252800017595291,148.95,148.95,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.29183998703956604,114.98,114.98,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,4.859903812408447,6.9,6.9,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.532480001449585,252.06,220.55,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5375999808311462,249.66,218.45,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,5.323775768280029,6.3,6.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12595200538635254,66.6,66.6,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.3686400055885315,91.02,91.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10342399775981903,81.11,81.11,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1443839967250824,58.1,58.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.27750399708747864,120.92,120.92,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5181440114974976,259.04,226.66,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12492799758911133,67.15,67.15,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08191999793052673,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08089599758386612,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.09113600105047226,0.09,0.16,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07782399654388428,0.11,0.18,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07577600330114365,0.11,0.19,1 +CommType.all_reduce,CommGroup.dp_group,4,4,forward_step.average_losses_across_data_parallel_group,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.158720001578331,105.7,92.49,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.16998399794101715,98.7,86.36,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1525759994983673,109.96,96.21,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.17100800573825836,98.11,85.84,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronEmbedding,None,None,,0.1884160041809082,89.04,155.83,1 +CommType.reduce_scatter,CommGroup.dp_group,4,35634282496,step,None,None,,129.49913024902344,275.17,240.77,1 +CommType.all_gather,CommGroup.dp_group,4,17817141248,step,None,None,,63.0015983581543,282.8,247.45,1 +CommType.all_reduce,CommGroup.tp_group,2,0,step._allreduce_layernorm_grads,None,None,,0.15667200088500977,0.0,0.0,1 +CommType.all_reduce,CommGroup.tp_group,2,4,step.check_for_nan,None,None,,0.4853760004043579,0.0,0.0,1 +CommType.epoch_end,None,None,0,,None,None,,444.4317817687988,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,0.09932799637317657,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,16448,forward_step,None,0,,0.0798719972372055,0.21,0.21,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronEmbedding,None,None,,0.187391996383667,89.53,156.68,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08191999793052673,0.8,0.7,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.13312000036239624,63.02,63.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2242559939622879,149.63,149.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2314240038394928,144.99,144.99,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.130048006772995,64.5,64.5,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22220799326896667,151.0,151.0,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5181440114974976,259.04,226.66,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2252800017595291,148.95,148.95,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12390399724245071,67.7,67.7,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21913599967956543,153.12,153.12,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08191999793052673,0.8,0.7,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.6748160123825073,198.9,174.03,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.3184640109539032,105.36,105.36,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20070399343967438,167.18,167.18,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20070399343967438,167.18,167.18,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08191999793052673,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.13619199395179749,61.59,61.59,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2242559939622879,149.63,149.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08499199897050858,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.5468159914016724,61.36,61.36,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,4.137983798980713,8.11,8.11,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21913599967956543,153.12,153.12,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.16486400365829468,101.76,89.04,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.13209599256515503,63.5,63.5,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5017600059509277,267.49,234.06,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08396799862384796,0.78,0.68,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22118400037288666,151.7,151.7,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.13414399325847626,62.53,62.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5181440114974976,259.04,226.66,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.52019202709198,258.02,225.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.0942080020904541,0.09,0.15,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.08191999793052673,0.1,0.18,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07680000364780426,0.11,0.19,1 +CommType.all_reduce,CommGroup.dp_group,4,4,forward_step.average_losses_across_data_parallel_group,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.16179199516773224,103.7,90.73,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.2242559939622879,74.81,65.46,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.17100800573825836,98.11,85.84,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14950400590896606,112.22,98.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14950400590896606,112.22,98.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronEmbedding,None,None,,0.18636800348758698,90.02,157.54,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,0.0870399996638298,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,16448,forward_step,None,0,,0.07782399654388428,0.21,0.21,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronEmbedding,None,None,,0.17817600071430206,94.16,164.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.14233599603176117,58.94,58.94,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.24883200228214264,134.85,134.85,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2242559939622879,149.63,149.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07168000191450119,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2017280012369156,166.34,166.34,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22220799326896667,151.0,151.0,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12390399724245071,67.7,67.7,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1730560064315796,96.95,84.83,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07475200295448303,0.88,0.77,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11980800330638885,70.02,70.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2017280012369156,166.34,166.34,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11980800330638885,70.02,70.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12083200365304947,69.42,69.42,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22835199534893036,146.94,146.94,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5253120064735413,255.5,223.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2314240038394928,144.99,144.99,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08191999793052673,0.8,0.7,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22220799326896667,151.0,151.0,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22118400037288666,151.7,151.7,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2242559939622879,149.63,149.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11980800330638885,70.02,70.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14950400590896606,112.22,98.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.15564799308776855,107.79,94.32,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.52019202709198,258.02,225.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.15564799308776855,107.79,94.32,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12595200538635254,66.6,66.6,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14950400590896606,112.22,98.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07168000191450119,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21913599967956543,153.12,153.12,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12697599828243256,66.06,66.06,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07168000191450119,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12697599828243256,66.06,66.06,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.0942080020904541,0.09,0.15,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.08089599758386612,0.1,0.18,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07680000364780426,0.11,0.19,1 +CommType.all_reduce,CommGroup.dp_group,4,4,forward_step.average_losses_across_data_parallel_group,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12697599828243256,132.13,115.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.16486400365829468,101.76,89.04,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1515520066022873,110.7,96.86,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15667200088500977,107.08,93.7,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronEmbedding,None,None,,0.1955839991569519,85.78,150.12,1 +CommType.reduce_scatter,CommGroup.dp_group,4,35634282496,step,None,None,,128.26010131835938,277.83,243.1,1 +CommType.all_gather,CommGroup.dp_group,4,17817141248,step,None,None,,63.43782424926758,280.86,245.75,1 +CommType.all_reduce,CommGroup.tp_group,2,0,step._allreduce_layernorm_grads,None,None,,0.16486400365829468,0.0,0.0,1 +CommType.all_reduce,CommGroup.tp_group,2,4,step.check_for_nan,None,None,,0.08601599931716919,0.0,0.0,1 +CommType.epoch_end,None,None,0,,None,None,,437.20126152038574,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,0.09728000313043594,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,16448,forward_step,None,0,,0.0798719972372055,0.21,0.21,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronEmbedding,None,None,,0.18636800348758698,90.02,157.54,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.131071999669075,64.0,64.0,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2314240038394928,144.99,144.99,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5017600059509277,267.49,234.06,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2672640085220337,125.55,125.55,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5355520248413086,250.62,219.29,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1730560064315796,96.95,84.83,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12697599828243256,66.06,66.06,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08089599758386612,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.13516800105571747,62.06,62.06,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.13209599256515503,63.5,63.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.15360000729560852,109.23,95.57,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08396799862384796,0.78,0.68,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5017600059509277,267.49,234.06,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07680000364780426,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,5.494783878326416,6.11,6.11,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.2549760043621063,65.8,57.57,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22015999257564545,152.41,152.41,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07168000191450119,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5017600059509277,267.49,234.06,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.08908800035715103,0.09,0.16,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07884799689054489,0.1,0.18,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07577600330114365,0.11,0.19,1 +CommType.all_reduce,CommGroup.dp_group,4,4,forward_step.average_losses_across_data_parallel_group,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.17612800002098083,95.26,83.35,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1730560064315796,96.95,84.83,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronEmbedding,None,None,,0.2017280012369156,83.17,145.54,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,0.0870399996638298,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,16448,forward_step,None,0,,0.07577600330114365,0.22,0.22,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronEmbedding,None,None,,0.17715199291706085,94.71,165.73,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.14131200313568115,59.36,59.36,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2529279887676239,132.66,132.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2017280012369156,166.34,166.34,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1669120043516159,100.52,87.95,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.09625600278377533,0.68,0.6,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.10649599879980087,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.13619199395179749,61.59,61.59,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22630399465560913,148.27,148.27,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08294399827718735,0.79,0.69,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11980800330638885,70.02,70.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.3164159953594208,106.05,106.05,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2017280012369156,166.34,166.34,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2252800017595291,148.95,148.95,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.23449599742889404,143.09,143.09,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5212159752845764,257.51,225.32,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21913599967956543,153.12,153.12,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.19660800695419312,85.33,74.67,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.14028799533843994,59.8,59.8,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.15360000729560852,109.23,95.57,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22220799326896667,151.0,151.0,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.158720001578331,105.7,92.49,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.23244799673557281,144.35,144.35,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.09011200070381165,0.09,0.16,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.08089599758386612,0.1,0.18,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07680000364780426,0.11,0.19,1 +CommType.all_reduce,CommGroup.dp_group,4,4,forward_step.average_losses_across_data_parallel_group,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14950400590896606,112.22,98.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.18432000279426575,91.02,79.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.18636800348758698,90.02,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronEmbedding,None,None,,0.20070399343967438,83.59,146.29,1 +CommType.reduce_scatter,CommGroup.dp_group,4,35634282496,step,None,None,,128.34303283691406,277.65,242.94,1 +CommType.all_gather,CommGroup.dp_group,4,17817141248,step,None,None,,63.947776794433594,278.62,243.79,1 +CommType.all_reduce,CommGroup.tp_group,2,0,step._allreduce_layernorm_grads,None,None,,0.15462400019168854,0.0,0.0,1 +CommType.all_reduce,CommGroup.tp_group,2,4,step.check_for_nan,None,None,,0.0870399996638298,0.0,0.0,1 +CommType.epoch_end,None,None,0,,None,None,,439.164400100708,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,0.09830400347709656,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,16448,forward_step,None,0,,0.08191999793052673,0.2,0.2,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronEmbedding,None,None,,0.18534399569034576,90.52,158.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08806400001049042,0.74,0.65,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.15769599378108978,53.19,53.19,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2303999960422516,145.64,145.64,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08806400001049042,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20070399343967438,167.18,167.18,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1228799968957901,68.27,68.27,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07680000364780426,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1740799993276596,96.38,84.33,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08601599931716919,0.76,0.67,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.13414399325847626,62.53,62.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.52019202709198,258.02,225.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11980800330638885,70.02,70.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5017600059509277,267.49,234.06,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21913599967956543,153.12,153.12,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.4034560024738312,83.17,83.17,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20070399343967438,167.18,167.18,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,2.2896640300750732,58.62,51.29,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,5.470208168029785,24.54,21.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2519040107727051,133.2,133.2,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21913599967956543,153.12,153.12,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.634880006313324,211.41,184.98,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07168000191450119,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22630399465560913,148.27,148.27,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5232639908790588,256.5,224.44,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5191680192947388,258.52,226.21,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12492799758911133,67.15,67.15,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2017280012369156,166.34,166.34,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5007359981536865,268.04,234.54,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08089599758386612,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22835199534893036,146.94,146.94,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.23244799673557281,144.35,144.35,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5212159752845764,257.51,225.32,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.19353599846363068,86.69,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2303999960422516,145.64,145.64,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1382399946451187,60.68,60.68,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22323200106620789,150.31,150.31,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.09113600105047226,0.09,0.16,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.08089599758386612,0.1,0.18,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07782399654388428,0.11,0.18,1 +CommType.all_reduce,CommGroup.dp_group,4,4,forward_step.average_losses_across_data_parallel_group,None,None,,0.08294399827718735,0.0,0.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15769599378108978,106.39,93.09,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.16998399794101715,98.7,86.36,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.17510400712490082,95.81,83.84,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.16076800227165222,104.36,91.31,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.16076800227165222,104.36,91.31,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12697599828243256,132.13,115.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.16998399794101715,98.7,86.36,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronEmbedding,None,None,,0.18636800348758698,90.02,157.54,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,0.08908800035715103,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,16448,forward_step,None,0,,0.07680000364780426,0.21,0.21,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronEmbedding,None,None,,0.1884160041809082,89.04,155.83,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.13619199395179749,61.59,61.59,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.23654399812221527,141.85,141.85,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22323200106620789,150.31,150.31,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.13414399325847626,62.53,62.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21913599967956543,153.12,153.12,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.15360000729560852,109.23,95.57,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12800000607967377,65.54,65.54,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11980800330638885,70.02,70.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21913599967956543,153.12,153.12,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22118400037288666,151.7,151.7,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.3256320059299469,103.04,103.04,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07168000191450119,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12083200365304947,69.42,69.42,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12595200538635254,66.6,66.6,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22118400037288666,151.7,151.7,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08089599758386612,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22015999257564545,152.41,152.41,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2314240038394928,144.99,144.99,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1515520066022873,110.7,96.86,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08089599758386612,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12800000607967377,65.54,65.54,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12697599828243256,66.06,66.06,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,4.343808174133301,7.72,7.72,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07680000364780426,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22220799326896667,151.0,151.0,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2252800017595291,148.95,148.95,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12083200365304947,69.42,69.42,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22323200106620789,150.31,150.31,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5181440114974976,259.04,226.66,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.23449599742889404,143.09,143.09,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22015999257564545,152.41,152.41,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5222399830818176,257.0,224.88,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.08908800035715103,0.09,0.16,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.0798719972372055,0.1,0.18,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07577600330114365,0.11,0.19,1 +CommType.all_reduce,CommGroup.dp_group,4,4,forward_step.average_losses_across_data_parallel_group,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1525759994983673,109.96,96.21,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.16896000504493713,99.3,86.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1525759994983673,109.96,96.21,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronEmbedding,None,None,,0.1884160041809082,89.04,155.83,1 +CommType.reduce_scatter,CommGroup.dp_group,4,35634282496,step,None,None,,128.09829711914062,278.18,243.41,1 +CommType.all_gather,CommGroup.dp_group,4,17817141248,step,None,None,,63.60166549682617,280.14,245.12,1 +CommType.all_reduce,CommGroup.tp_group,2,0,step._allreduce_layernorm_grads,None,None,,0.15462400019168854,0.0,0.0,1 +CommType.all_reduce,CommGroup.tp_group,2,4,step.check_for_nan,None,None,,0.08499199897050858,0.0,0.0,1 +CommType.epoch_end,None,None,0,,None,None,,444.5972442626953,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,0.09728000313043594,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,16448,forward_step,None,0,,0.08089599758386612,0.2,0.2,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronEmbedding,None,None,,0.187391996383667,89.53,156.68,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.13312000036239624,63.02,63.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22118400037288666,151.7,151.7,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07168000191450119,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2252800017595291,148.95,148.95,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5468159914016724,245.45,214.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12697599828243256,66.06,66.06,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07168000191450119,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22323200106620789,150.31,150.31,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5212159752845764,257.51,225.32,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14950400590896606,112.22,98.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2017280012369156,166.34,166.34,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.23244799673557281,144.35,144.35,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.13516800105571747,62.06,62.06,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.17715199291706085,94.71,82.87,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12492799758911133,67.15,67.15,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1228799968957901,68.27,68.27,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21913599967956543,153.12,153.12,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07168000191450119,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.13414399325847626,62.53,62.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.15667200088500977,107.08,93.7,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0870399996638298,0.75,0.66,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08089599758386612,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08294399827718735,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21913599967956543,153.12,153.12,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12083200365304947,69.42,69.42,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1740799993276596,96.38,84.33,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22835199534893036,146.94,146.94,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21913599967956543,153.12,153.12,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08294399827718735,0.79,0.69,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08396799862384796,0.78,0.68,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07577600330114365,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08396799862384796,0.78,0.68,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08294399827718735,0.79,0.69,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.08908800035715103,0.09,0.16,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07680000364780426,0.11,0.19,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07372800260782242,0.11,0.19,1 +CommType.all_reduce,CommGroup.dp_group,4,4,forward_step.average_losses_across_data_parallel_group,None,None,,0.08089599758386612,0.0,0.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15564799308776855,107.79,94.32,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.159743994474411,105.03,91.9,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12697599828243256,132.13,115.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronEmbedding,None,None,,0.18636800348758698,90.02,157.54,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,0.08908800035715103,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,16448,forward_step,None,0,,0.0798719972372055,0.21,0.21,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronEmbedding,None,None,,0.19046400487422943,88.09,154.15,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1515520066022873,110.7,96.86,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.131071999669075,64.0,64.0,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2529279887676239,132.66,132.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11980800330638885,70.02,70.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22118400037288666,151.7,151.7,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2027519941329956,165.49,165.49,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12902399897575378,65.02,65.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22323200106620789,150.31,150.31,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12083200365304947,69.42,69.42,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11980800330638885,70.02,70.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22015999257564545,152.41,152.41,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08294399827718735,0.79,0.69,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5212159752845764,257.51,225.32,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.13209599256515503,63.5,63.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07577600330114365,0.86,0.76,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1228799968957901,68.27,68.27,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1382399946451187,60.68,60.68,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22015999257564545,152.41,152.41,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08191999793052673,0.8,0.7,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21913599967956543,153.12,153.12,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07168000191450119,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22015999257564545,152.41,152.41,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.16179199516773224,103.7,90.73,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22015999257564545,152.41,152.41,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,5.119999885559082,6.55,6.55,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1228799968957901,68.27,68.27,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.22630399465560913,37.07,37.07,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.502784013748169,266.95,233.58,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07782399654388428,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08089599758386612,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.09113600105047226,0.09,0.16,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.08089599758386612,0.1,0.18,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.07577600330114365,0.11,0.19,1 +CommType.all_reduce,CommGroup.dp_group,4,4,forward_step.average_losses_across_data_parallel_group,None,None,,0.07884799689054489,0.0,0.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.158720001578331,105.7,92.49,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.16793599724769592,99.9,87.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12697599828243256,132.13,115.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12697599828243256,132.13,115.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.16383999586105347,102.4,89.6,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.12902399897575378,130.03,113.78,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1658879965543747,101.14,88.49,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.12800000607967377,131.07,114.69,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.130048006772995,129.01,112.88,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronEmbedding,None,None,,0.1955839991569519,85.78,150.12,1 +CommType.reduce_scatter,CommGroup.dp_group,4,35634282496,step,None,None,,129.17657470703125,275.86,241.38,1 +CommType.all_gather,CommGroup.dp_group,4,17817141248,step,None,None,,63.86278533935547,278.99,244.12,1 +CommType.all_reduce,CommGroup.tp_group,2,0,step._allreduce_layernorm_grads,None,None,,0.11673600226640701,0.0,0.0,1 +CommType.all_reduce,CommGroup.tp_group,2,4,step.check_for_nan,None,None,,0.08191999793052673,0.0,0.0,1 +CommType.epoch_end,None,None,0,,None,None,,438.6782646179199,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,0.09830400347709656,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,16448,forward_step,None,0,,0.08191999793052673,0.2,0.2,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronEmbedding,None,None,,0.18534399569034576,90.52,158.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08396799862384796,0.78,0.68,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.131071999669075,64.0,64.0,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22323200106620789,150.31,150.31,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,3.221503973007202,10.42,10.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.1157120019197464,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.15462400019168854,54.25,54.25,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2314240038394928,144.99,144.99,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5222399830818176,257.0,224.88,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.536575973033905,250.14,218.87,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,5.080063819885254,6.61,6.61,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12390399724245071,67.7,67.7,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08089599758386612,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22732800245285034,147.6,147.6,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5232639908790588,256.5,224.44,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5232639908790588,256.5,224.44,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12083200365304947,69.42,69.42,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1228799968957901,68.27,68.27,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5017600059509277,267.49,234.06,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1228799968957901,68.27,68.27,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11059200018644333,75.85,75.85,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2979840040206909,112.6,112.6,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08191999793052673,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.15462400019168854,54.25,54.25,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22732800245285034,147.6,147.6,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5048320293426514,265.87,232.63,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07680000364780426,0.85,0.75,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07680000364780426,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21913599967956543,153.12,153.12,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.43110400438308716,77.83,77.83,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08089599758386612,0.81,0.71,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20479999482631683,163.84,163.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20787200331687927,161.42,161.42,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2017280012369156,166.34,166.34,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20582400262355804,163.02,163.02,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10547199845314026,79.53,79.53,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0727040022611618,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20377600193023682,164.66,164.66,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10444799810647964,80.31,80.31,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11980800330638885,70.02,70.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2242559939622879,149.63,149.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.26521599292755127,126.52,126.52,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07372800260782242,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10649599879980087,78.77,78.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.8663039803504944,38.73,38.73,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12595200538635254,66.6,66.6,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11161600053310394,75.16,75.16,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0798719972372055,0.82,0.72,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10751999914646149,78.02,78.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5038080215454102,266.41,233.11,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.4208639860153198,79.73,79.73,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1085439994931221,77.28,77.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.15360000729560852,109.23,95.57,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07884799689054489,0.83,0.73,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.07475200295448303,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5058559775352478,265.33,232.16,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.20684799551963806,162.22,162.22,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.10956799983978271,76.56,76.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.07782399654388428,0.84,0.74,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,4.336639881134033,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.18329599499702454,45.77,45.77,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.24780799448490143,135.4,135.4,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.52019202709198,258.02,225.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5191680192947388,258.52,226.21,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,3.808255910873413,8.81,8.81,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12595200538635254,66.6,66.6,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.09011200070381165,0.73,0.64,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08601599931716919,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5191680192947388,258.52,226.21,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.09011200070381165,0.73,0.64,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08294399827718735,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5191680192947388,258.52,226.21,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14950400590896606,112.22,98.19,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08806400001049042,0.74,0.65,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08294399827718735,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.09011200070381165,0.73,0.64,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08396799862384796,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12595200538635254,66.6,66.6,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22220799326896667,151.0,151.0,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5191680192947388,258.52,226.21,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0870399996638298,0.75,0.66,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08499199897050858,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.131071999669075,64.0,64.0,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,6.552576065063477,5.12,5.12,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5181440114974976,259.04,226.66,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5191680192947388,258.52,226.21,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.09932799637317657,0.08,0.14,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.08806400001049042,0.09,0.16,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.0870399996638298,0.09,0.16,1 +CommType.all_reduce,CommGroup.dp_group,4,4,forward_step.average_losses_across_data_parallel_group,None,None,,0.09011200070381165,0.0,0.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.16076800227165222,104.36,91.31,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15360000729560852,109.23,95.57,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.16076800227165222,104.36,91.31,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14950400590896606,112.22,98.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.20787200331687927,80.71,70.62,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15360000729560852,109.23,95.57,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.15769599378108978,106.39,93.09,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14950400590896606,112.22,98.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronEmbedding,None,None,,0.1945600062608719,86.23,150.91,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,0.09728000313043594,0.0,0.0,1 +CommType.broadcast,CommGroup.tp_group,2,16448,forward_step,None,0,,0.0870399996638298,0.19,0.19,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronEmbedding,None,None,,0.19353599846363068,86.69,151.7,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.09011200070381165,0.73,0.64,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08601599931716919,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.15462400019168854,54.25,54.25,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.26214399933815,128.0,128.0,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.52019202709198,258.02,225.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22118400037288666,151.7,151.7,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12595200538635254,66.6,66.6,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.09011200070381165,0.73,0.64,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08499199897050858,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08806400001049042,0.74,0.65,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08191999793052673,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.130048006772995,64.5,64.5,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2314240038394928,144.99,144.99,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.766975998878479,175.0,153.12,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5212159752845764,257.51,225.32,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22015999257564545,152.41,152.41,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.19968000054359436,84.02,73.52,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.09318400174379349,0.7,0.62,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0870399996638298,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,6.564864158630371,1.28,1.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.15360000729560852,109.23,95.57,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08908800035715103,0.74,0.64,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08191999793052673,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22323200106620789,150.31,150.31,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5191680192947388,258.52,226.21,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12390399724245071,67.7,67.7,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0870399996638298,0.75,0.66,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08191999793052673,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.14131200313568115,59.36,59.36,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22630399465560913,148.27,148.27,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12083200365304947,69.42,69.42,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.09011200070381165,0.73,0.64,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08191999793052673,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08908800035715103,0.74,0.64,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08294399827718735,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.09011200070381165,0.73,0.64,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08294399827718735,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08908800035715103,0.74,0.64,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08294399827718735,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08908800035715103,0.74,0.64,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08191999793052673,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12185599654912949,68.84,68.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5181440114974976,259.04,226.66,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08806400001049042,0.74,0.65,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08191999793052673,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12595200538635254,66.6,66.6,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.24780799448490143,135.4,135.4,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.536575973033905,250.14,218.87,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14847999811172485,112.99,98.87,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.09216000139713287,0.71,0.62,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08499199897050858,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22732800245285034,147.6,147.6,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5130239725112915,261.62,228.92,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0870399996638298,0.75,0.66,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08499199897050858,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.23654399812221527,141.85,141.85,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5191680192947388,258.52,226.21,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1228799968957901,68.27,68.27,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.15360000729560852,109.23,95.57,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08908800035715103,0.74,0.64,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08089599758386612,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12595200538635254,66.6,66.6,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.23347200453281403,143.72,143.72,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.15360000729560852,109.23,95.57,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0870399996638298,0.75,0.66,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08191999793052673,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12083200365304947,69.42,69.42,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5212159752845764,257.51,225.32,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0870399996638298,0.75,0.66,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08089599758386612,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0870399996638298,0.75,0.66,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08396799862384796,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2181120067834854,153.84,153.84,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5079039931297302,264.26,231.23,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08806400001049042,0.74,0.65,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08191999793052673,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11264000087976456,74.47,74.47,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1730560064315796,96.95,84.83,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08601599931716919,0.76,0.67,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08396799862384796,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22015999257564545,152.41,152.41,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5181440114974976,259.04,226.66,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08499199897050858,0.77,0.67,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08499199897050858,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11980800330638885,70.02,70.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22937600314617157,146.29,146.29,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5273600220680237,254.51,222.7,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12390399724245071,67.7,67.7,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1525759994983673,109.96,96.21,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08806400001049042,0.74,0.65,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08089599758386612,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1228799968957901,68.27,68.27,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.25088000297546387,133.75,133.75,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12390399724245071,67.7,67.7,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08601599931716919,0.76,0.67,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08191999793052673,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.22323200106620789,150.31,150.31,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5140479803085327,261.1,228.46,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5171200037002563,259.55,227.1,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11673600226640701,71.86,71.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.09523200243711472,0.69,0.6,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08294399827718735,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21094399690628052,159.07,159.07,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2088959962129593,160.63,160.63,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11980800330638885,70.02,70.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08908800035715103,0.74,0.64,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08191999793052673,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11366400122642517,73.8,73.8,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.506879985332489,264.79,231.69,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5191680192947388,258.52,226.21,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08908800035715103,0.74,0.64,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08191999793052673,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21913599967956543,153.12,153.12,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5099520087242126,263.2,230.3,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5150719881057739,260.58,228.01,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08806400001049042,0.74,0.65,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08191999793052673,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11468800157308578,73.14,73.14,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5191680192947388,258.52,226.21,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21401600539684296,156.78,156.78,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12083200365304947,69.42,69.42,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08601599931716919,0.76,0.67,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.0798719972372055,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12185599654912949,68.84,68.84,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21606400609016418,155.3,155.3,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5120000243186951,262.14,229.38,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21196800470352173,158.3,158.3,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1157120019197464,72.5,72.5,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08499199897050858,0.77,0.67,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08601599931716919,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.24371199309825897,137.68,137.68,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.6707199811935425,200.11,175.1,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.52019202709198,258.02,225.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.1228799968957901,68.27,68.27,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.16179199516773224,103.7,90.73,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.0870399996638298,0.75,0.66,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08396799862384796,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.1525759994983673,54.98,54.98,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.23756800591945648,141.24,141.24,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5109760165214539,262.67,229.84,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2252800017595291,148.95,148.95,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11980800330638885,70.02,70.02,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08499199897050858,0.77,0.67,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08601599931716919,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.12902399897575378,65.02,65.02,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2170879989862442,154.57,154.57,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5160959959030151,260.06,227.56,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21299199759960175,157.54,157.54,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.12390399724245071,67.7,67.7,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,forward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,0.1515520066022873,110.7,96.86,1 +CommType.all_gather,CommGroup.tp_group,2,65536,forward.MoE,None,None,,0.08908800035715103,0.74,0.64,1 +CommType.all_gather,CommGroup.tp_group,2,32,forward.MoE,None,None,,0.08191999793052673,0.0,0.0,1 +CommType.all_to_all,CommGroup.tp_group,2,8388608,forward.MoE,None,None,,0.11776000261306763,71.23,71.23,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.21503999829292297,156.04,156.04,1 +CommType.all_gather,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.5089280009269714,263.73,230.76,1 +CommType.reduce_scatter,CommGroup.tp_group,None,134217728,forward.MoE,None,None,,0.52019202709198,258.02,225.76,1 +CommType.all_to_all,CommGroup.ep_group,None,33554432,forward.MoE,None,None,,0.2099200040102005,159.84,159.84,1 +CommType.all_to_all,CommGroup.tp_group,None,8388608,forward.MoE,None,None,,0.11878400295972824,70.62,70.62,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.10035199671983719,0.08,0.14,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.08908800035715103,0.09,0.16,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,0.08601599931716919,0.1,0.17,1 +CommType.all_reduce,CommGroup.dp_group,4,4,forward_step.average_losses_across_data_parallel_group,None,None,,0.09523200243711472,0.0,0.0,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14233599603176117,117.87,103.14,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1382399946451187,121.36,106.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.16179199516773224,103.7,90.73,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.16383999586105347,102.4,89.6,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1515520066022873,110.7,96.86,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1525759994983673,109.96,96.21,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1669120043516159,100.52,87.95,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1658879965543747,101.14,88.49,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13312000036239624,126.03,110.28,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13209599256515503,127.01,111.13,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14950400590896606,112.22,98.19,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.131071999669075,128.0,112.0,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1443839967250824,116.2,101.67,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1454080045223236,115.38,100.96,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14028799533843994,119.59,104.64,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15360000729560852,109.23,95.57,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13516800105571747,124.12,108.61,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13926400244235992,120.47,105.41,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13619199395179749,123.19,107.79,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1525759994983673,109.96,96.21,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14336000382900238,117.03,102.4,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.1372160017490387,122.27,106.99,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.15052799880504608,111.46,97.52,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.13414399325847626,125.07,109.44,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14643199741840363,114.57,100.25,1 +CommType.reduce_scatter,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,0.14131200313568115,118.72,103.88,1 +CommType.all_gather,CommGroup.tp_group,2,16777216,backward.MegatronRowLinear,None,None,,0.14745600521564484,113.78,99.56,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronEmbedding,None,None,,0.19251200556755066,87.15,152.51,1 +CommType.reduce_scatter,CommGroup.dp_group,4,35634282496,step,None,None,,128.96563720703125,276.31,241.77,1 +CommType.all_gather,CommGroup.dp_group,4,17817141248,step,None,None,,63.356929779052734,281.22,246.07,1 +CommType.all_reduce,CommGroup.tp_group,2,0,step._allreduce_layernorm_grads,None,None,,0.1443839967250824,0.0,0.0,1 +CommType.all_reduce,CommGroup.tp_group,2,4,step.check_for_nan,None,None,,0.0870399996638298,0.0,0.0,1 +CommType.epoch_end,None,None,0,,None,None,,473.88172149658203,0.0,0.0,1 diff --git a/aicb/workload/Workload_spec_v1.1.csv b/aicb/workload/Workload_spec_v1.1.csv new file mode 100644 index 000000000..b995d1d84 --- /dev/null +++ b/aicb/workload/Workload_spec_v1.1.csv @@ -0,0 +1,11 @@ +id Name Parameter_size Hidden_size Num_of_layers Attention_heads Sequence_length FFN_hidden_size Name World_size TP DP PP SP Zero_level expert parallel number Expert num TopK group_gemm reduce_bucket_size allgather_bucket_size prefetch_bucket_size max_live_parameters param_persistence_threshold +1 LLaMA_7B 7B 4096 32 32 2048 16384 Megatron 128 1 world_size/(PP*TP) 1 - - - - - - - - - - - +2 GPT_13B 13B 5120 40 32 2048 20480 Megatron 128 2 world_size/(PP*TP) 1 enable - - - - - - - - - - +3 GPT_22B 22B 6144 48 64 2048 24576 Megatron 128 4 world_size/(PP*TP) 1 - - - - - - - - - - - +4 LLaMA_65B 65B 8192 80 64 4096 28672 Megatron 128 8 world_size/(PP*TP) 2 enable - - - - - - - - - - +5 GPT_175B 175B 12288 96 96 2048 49152 Megatron 128 8 world_size/(PP*TP) 8 enable - - - - - - - - - - +6 GPT_175B 175B 12288 96 96 2048 49152 Megatron 128 8 world_size/(PP*TP) 8 disable - - - - - - - - - - +7 Llama3 405B 405B 16384 126 128 8192 53248 Megetron 128 8 world_size/(PP*TP) 16 enable - - - - - - - - +8 LLaMA_7B 7B 4096 32 32 4096 11008 Deepspeed 128 1 world_size 1 - 2 - - - - 1.00E+09 1.00E+09 - - - +9 LLaMA_65B 65B 8192 80 64 4096 28672 Deepspeed 128 1 world_size 1 - 3 - - - - 1.00E+09 - 1.00E+09 6.00E+08 1.00E+06 +10 Mistral_8*7B 56B 4096 32 32 2048 14336 Megatron 128 2 world_size/(PP*TP) 1 enable - 8 8 2 true - - - - - \ No newline at end of file diff --git a/aicb/workload/aiob_inputs/Example.txt b/aicb/workload/aiob_inputs/Example.txt new file mode 100644 index 000000000..ae7715060 --- /dev/null +++ b/aicb/workload/aiob_inputs/Example.txt @@ -0,0 +1,49 @@ +train_iter:10 +Emb: + time_gpu_max: 857 + time_gpu_min: 782 + time_gpu_avg: 799 +layernorm: + time_gpu_max: 84 + time_gpu_min: 43 + time_gpu_avg: 70 +atten_qkv: + time_gpu_max: 1255 + time_gpu_min: 862 + time_gpu_avg: 889 +atten_flash: + time_gpu_max: 786 + time_gpu_min: 460 + time_gpu_avg: 512 +atten_linear: + time_gpu_max: 939 + time_gpu_min: 333 + time_gpu_avg: 349 +layernorm2: + time_gpu_max: 116 + time_gpu_min: 67 + time_gpu_avg: 71 +mlp_linear_1: + time_gpu_max: 1876 + time_gpu_min: 1372 + time_gpu_avg: 1408 +mlp_gelu: + time_gpu_max: 560 + time_gpu_min: 246 + time_gpu_avg: 257 +mlp_linear_2: + time_gpu_max: 1183 + time_gpu_min: 720 + time_gpu_avg: 742 +layernorm_post: + time_gpu_max: 73 + time_gpu_min: 68 + time_gpu_avg: 70 +logit_time: + time_gpu_max: 15476 + time_gpu_min: 12435 + time_gpu_avg: 12824 +param_time: + time_gpu_max: 23746 + time_gpu_min: 16647 + time_gpu_avg: 17374 diff --git a/aicb/workload/physical/micro_test/all_gather_workload.csv b/aicb/workload/physical/micro_test/all_gather_workload.csv new file mode 100644 index 000000000..5a8ffd8c5 --- /dev/null +++ b/aicb/workload/physical/micro_test/all_gather_workload.csv @@ -0,0 +1,2303 @@ +comm_type,comm_group,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.epoch_end,None,0,,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.epoch_end,None,0,,None,None,,None,None,None,1 diff --git a/aicb/workload/physical/micro_test/all_reduce_workload.csv b/aicb/workload/physical/micro_test/all_reduce_workload.csv new file mode 100644 index 000000000..6bc4c81df --- /dev/null +++ b/aicb/workload/physical/micro_test/all_reduce_workload.csv @@ -0,0 +1,2303 @@ +comm_type,comm_group,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.epoch_end,None,0,,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.epoch_end,None,0,,None,None,,None,None,None,1 diff --git a/aicb/workload/physical/micro_test/all_to_all_workload.csv b/aicb/workload/physical/micro_test/all_to_all_workload.csv new file mode 100644 index 000000000..d674a1943 --- /dev/null +++ b/aicb/workload/physical/micro_test/all_to_all_workload.csv @@ -0,0 +1,2003 @@ +comm_type,comm_group,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.epoch_end,None,0,,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_to_all,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.epoch_end,None,0,,None,None,,None,None,None,1 diff --git a/aicb/workload/physical/micro_test/multi_all_reduce_workload.csv b/aicb/workload/physical/micro_test/multi_all_reduce_workload.csv new file mode 100644 index 000000000..9f0ce98c4 --- /dev/null +++ b/aicb/workload/physical/micro_test/multi_all_reduce_workload.csv @@ -0,0 +1,2003 @@ +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,4096,warmup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4096,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8192,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16384,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,32768,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,65536,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,131072,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,262144,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,524288,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1048576,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,2097152,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,4194304,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,8388608,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,16777216,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,33554432,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,67108864,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,134217728,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,268435456,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,536870912,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,1,1073741824,test_step,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 diff --git a/aicb/workload/physical/micro_test/reduce_scatter_workload.csv b/aicb/workload/physical/micro_test/reduce_scatter_workload.csv new file mode 100644 index 000000000..68efb9cd5 --- /dev/null +++ b/aicb/workload/physical/micro_test/reduce_scatter_workload.csv @@ -0,0 +1,2303 @@ +comm_type,comm_group,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,warmup,None,None,,None,None,None,1 +CommType.epoch_end,None,0,,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4096,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8192,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16384,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,32768,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,65536,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,131072,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,262144,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,524288,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1048576,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2097152,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4194304,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8388608,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16777216,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,33554432,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,67108864,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,134217728,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,268435456,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,536870912,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,1073741824,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,2147483648,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,4294967296,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,8589934592,test_step,None,None,,None,None,None,1 +CommType.epoch_end,None,0,,None,None,,None,None,None,1 diff --git a/aicb/workload/physical/model_workload/G13B-M1-C01_GPT13B_megatron_tp8_pp1_mbs1.csv b/aicb/workload/physical/model_workload/G13B-M1-C01_GPT13B_megatron_tp8_pp1_mbs1.csv new file mode 100644 index 000000000..5551a998a --- /dev/null +++ b/aicb/workload/physical/model_workload/G13B-M1-C01_GPT13B_megatron_tp8_pp1_mbs1.csv @@ -0,0 +1,183 @@ +[rank0 ~ rank127] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,8192,forward_step._VocabParallelCrossEntropy,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,8192,forward_step._VocabParallelCrossEntropy,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,8192,forward_step._VocabParallelCrossEntropy,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16,4,forward_step.average_losses_across_data_parallel_group,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16,6422958080,step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16,3211479040,step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,0,step._allreduce_layernorm_grads,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,4,step.check_for_nan,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 diff --git a/aicb/workload/physical/model_workload/G13B-M1-C02_GPT13B_megatron_tp8_pp1_mbs1_sp.csv b/aicb/workload/physical/model_workload/G13B-M1-C02_GPT13B_megatron_tp8_pp1_mbs1_sp.csv new file mode 100644 index 000000000..c18c06210 --- /dev/null +++ b/aicb/workload/physical/model_workload/G13B-M1-C02_GPT13B_megatron_tp8_pp1_mbs1_sp.csv @@ -0,0 +1,423 @@ +[rank0 ~ rank127] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,forward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,8192,forward_step._VocabParallelCrossEntropy,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,8192,forward_step._VocabParallelCrossEntropy,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,8192,forward_step._VocabParallelCrossEntropy,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,16,4,forward_step.average_losses_across_data_parallel_group,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.tp_group,8,20971520,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.tp_group,8,20971520,backward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,20971520,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,16,6422958080,step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,16,3211479040,step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,1638400,step._allreduce_layernorm_grads,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,4,step.check_for_nan,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 diff --git a/aicb/workload/physical/model_workload/G175B-M1-C03_GPT175B_megatron_tp8_pp16_mbs1.csv b/aicb/workload/physical/model_workload/G175B-M1-C03_GPT175B_megatron_tp8_pp16_mbs1.csv new file mode 100644 index 000000000..dc55874a0 --- /dev/null +++ b/aicb/workload/physical/model_workload/G175B-M1-C03_GPT175B_megatron_tp8_pp16_mbs1.csv @@ -0,0 +1,100514 @@ +[ pp_stage0: [rank0, rank1, rank2, rank3, rank4, rank5, rank6, rank7] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_next,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +[ pp_stage10: [rank80, rank81, rank82, rank83, rank84, rank85, rank86, rank87] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_next,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +[ pp_stage11: [rank88, rank89, rank90, rank91, rank92, rank93, rank94, rank95] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_next,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +[ pp_stage12: [rank96, rank97, rank98, rank99, rank100, rank101, rank102, rank103] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_next,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +[ pp_stage13: [rank104, rank105, rank106, rank107, rank108, rank109, rank110, rank111] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_next,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +[ pp_stage14: [rank112, rank113, rank114, rank115, rank116, rank117, rank118, rank119] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_next,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +[ pp_stage15: [rank120, rank121, rank122, rank123, rank124, rank125, rank126, rank127] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.embedding_group,8,100663296,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.embedding_group,8,50331648,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +[ pp_stage1: [rank8, rank9, rank10, rank11, rank12, rank13, rank14, rank15] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_next,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +[ pp_stage2: [rank16, rank17, rank18, rank19, rank20, rank21, rank22, rank23] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_next,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +[ pp_stage3: [rank24, rank25, rank26, rank27, rank28, rank29, rank30, rank31] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_next,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +[ pp_stage4: [rank32, rank33, rank34, rank35, rank36, rank37, rank38, rank39] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_next,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +[ pp_stage5: [rank40, rank41, rank42, rank43, rank44, rank45, rank46, rank47] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_next,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +[ pp_stage6: [rank48, rank49, rank50, rank51, rank52, rank53, rank54, rank55] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_next,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +[ pp_stage7: [rank56, rank57, rank58, rank59, rank60, rank61, rank62, rank63] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_next,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +[ pp_stage8: [rank64, rank65, rank66, rank67, rank68, rank69, rank70, rank71] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_next,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +[ pp_stage9: [rank72, rank73, rank74, rank75, rank76, rank77, rank78, rank79] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,1,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,16,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,1,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,8,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,forward.GPTRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,forward_step,None,None,recv_next,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,forward_step,None,None,send_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,50331648,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.GPTColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,8,50331648,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,50331648,backward_step,None,None,send_prev,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 diff --git a/aicb/workload/physical/model_workload/L13B-D1-C03_Llama13B_zero2_mbs1.csv b/aicb/workload/physical/model_workload/L13B-D1-C03_Llama13B_zero2_mbs1.csv new file mode 100644 index 000000000..9390340fd --- /dev/null +++ b/aicb/workload/physical/model_workload/L13B-D1-C03_Llama13B_zero2_mbs1.csv @@ -0,0 +1,396 @@ +[rank0 ~ rank127] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.broadcast,CommGroup.dp_group,128,327680000,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,327680000,init,None,None,,None,None,None,1 +CommType.barrier,CommGroup.all,128,327680000,init.__init__,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1879623680,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1903226880,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1903226880,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1903226880,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1903226880,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1903226880,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1903226880,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1903226880,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1903226880,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1903226880,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1903226880,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1903226880,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1903226880,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1313382400,step.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1,step.has_overflow,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2002440704,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2002440704,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2002440704,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2002440704,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2002440704,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2002440704,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2002440704,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2002440704,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2002440704,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2002440704,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2002440704,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2002440704,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2002440704,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 diff --git a/aicb/workload/physical/model_workload/L13B-D1-C04_Llama13B_zero3_mbs1.csv b/aicb/workload/physical/model_workload/L13B-D1-C04_Llama13B_zero3_mbs1.csv new file mode 100644 index 000000000..57161480e --- /dev/null +++ b/aicb/workload/physical/model_workload/L13B-D1-C04_Llama13B_zero3_mbs1.csv @@ -0,0 +1,1556 @@ +[rank0 ~ rank127] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.broadcast,CommGroup.dp_group,128,327680000,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,52428800,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,141557760,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,10240,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,327680000,init._broadcast_model,None,0,,None,None,None,1 +CommType.barrier,CommGroup.all,128,0,init._create_fp16_partitions_with_defragmentation,None,None,,None,None,None,1 +CommType.barrier,CommGroup.all,128,0,init._setup_for_real_optimizer,None,None,,None,None,None,1 +CommType.barrier,CommGroup.all,128,0,init._setup_for_real_optimizer,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,327680000,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,327680000,init._allgather_params,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,327680000,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,327680000,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,327680000,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1879623680,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1903226880,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1903226880,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1903226880,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1903226880,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1903226880,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1903226880,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1903226880,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1903226880,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1903226880,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1903226880,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1903226880,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1903226880,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,141557760,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,52428800,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,327680000,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1313382400,step.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1,step.has_overflow,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,8,step.get_grad_norm_direct,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,10240,step.allgather_fn,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 diff --git a/aicb/workload/physical/model_workload/L30B-D1-C05_Llama30B_zero2_mbs1.csv b/aicb/workload/physical/model_workload/L30B-D1-C05_Llama30B_zero2_mbs1.csv new file mode 100644 index 000000000..55b600981 --- /dev/null +++ b/aicb/workload/physical/model_workload/L30B-D1-C05_Llama30B_zero2_mbs1.csv @@ -0,0 +1,615 @@ +[rank0 ~ rank127] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.broadcast,CommGroup.dp_group,128,425984000,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,425984000,init,None,None,,None,None,None,1 +CommType.barrier,CommGroup.all,128,425984000,init.__init__,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1973197824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1901645824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1901645824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1962974208,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1962987520,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1901632512,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1901645824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1901645824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1962974208,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1962987520,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1901632512,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1901645824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1901645824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1962974208,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1962987520,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1901632512,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1901645824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1901645824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1962974208,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1962987520,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1901632512,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1901645824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1901645824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1962974208,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1962987520,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1901632512,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1901645824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1901645824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1962974208,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1962987520,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1901632512,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1901645824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1901645824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1496082432,step.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1,step.has_overflow,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2033059072,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 diff --git a/aicb/workload/physical/model_workload/L30B-D1-C06_Llama30B_zero3_mbs1.csv b/aicb/workload/physical/model_workload/L30B-D1-C06_Llama30B_zero3_mbs1.csv new file mode 100644 index 000000000..1d837e991 --- /dev/null +++ b/aicb/workload/physical/model_workload/L30B-D1-C06_Llama30B_zero3_mbs1.csv @@ -0,0 +1,2336 @@ +[rank0 ~ rank127] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.broadcast,CommGroup.dp_group,128,425984000,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,88604672,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,238551040,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,13312,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,425984000,init._broadcast_model,None,0,,None,None,None,1 +CommType.barrier,CommGroup.all,128,0,init._create_fp16_partitions_with_defragmentation,None,None,,None,None,None,1 +CommType.barrier,CommGroup.all,128,0,init._setup_for_real_optimizer,None,None,,None,None,None,1 +CommType.barrier,CommGroup.all,128,0,init._setup_for_real_optimizer,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,425984000,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,425984000,init._allgather_params,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,425984000,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,425984000,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,425984000,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1973197824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1901645824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1901645824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1962974208,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1962987520,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1901632512,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1901645824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1901645824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1962974208,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1962987520,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1901632512,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1901645824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1901645824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1962974208,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1962987520,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1901632512,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1901645824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1901645824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1962974208,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1962987520,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1901632512,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1901645824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1901645824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1962974208,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1962987520,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1901632512,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1901645824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1901645824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1962974208,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1962987520,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1901632512,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1901645824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1901645824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,238551040,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,88604672,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,425984000,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1496082432,step.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1,step.has_overflow,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,8,step.get_grad_norm_direct,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,13312,step.allgather_fn,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 diff --git a/aicb/workload/physical/model_workload/L65B-D1-C07_Llama65B_zero2_mbs1.csv b/aicb/workload/physical/model_workload/L65B-D1-C07_Llama65B_zero2_mbs1.csv new file mode 100644 index 000000000..eb8bace23 --- /dev/null +++ b/aicb/workload/physical/model_workload/L65B-D1-C07_Llama65B_zero2_mbs1.csv @@ -0,0 +1,888 @@ +[rank0 ~ rank127] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.broadcast,CommGroup.dp_group,128,524288000,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,524288000,init,None,None,,None,None,None,1 +CommType.barrier,CommGroup.all,128,524288000,init.__init__,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1933606912,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1946189824,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1061175296,step.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1,step.has_overflow,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2009535744,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 diff --git a/aicb/workload/physical/model_workload/L65B-D1-C07_Llama65B_zero3_mbs1.csv b/aicb/workload/physical/model_workload/L65B-D1-C07_Llama65B_zero3_mbs1.csv new file mode 100644 index 000000000..fbf236561 --- /dev/null +++ b/aicb/workload/physical/model_workload/L65B-D1-C07_Llama65B_zero3_mbs1.csv @@ -0,0 +1,3143 @@ +[rank0 ~ rank127] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.broadcast,CommGroup.dp_group,128,524288000,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,134217728,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,469762048,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,16384,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,524288000,init._broadcast_model,None,0,,None,None,None,1 +CommType.barrier,CommGroup.all,128,0,init._create_fp16_partitions_with_defragmentation,None,None,,None,None,None,1 +CommType.barrier,CommGroup.all,128,0,init._setup_for_real_optimizer,None,None,,None,None,None,1 +CommType.barrier,CommGroup.all,128,0,init._setup_for_real_optimizer,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,524288000,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,524288000,init._allgather_params,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,524288000,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,524288000,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,524288000,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1933606912,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,469762048,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1946189824,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,134217728,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,524288000,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1061175296,step.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1,step.has_overflow,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,8,step.get_grad_norm_direct,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,16384,step.allgather_fn,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 diff --git a/aicb/workload/physical/model_workload/L65B-M1-C05_Llama65B_megatron_tp2_pp8_mbs1.csv b/aicb/workload/physical/model_workload/L65B-M1-C05_Llama65B_megatron_tp2_pp8_mbs1.csv new file mode 100644 index 000000000..f61303d8c --- /dev/null +++ b/aicb/workload/physical/model_workload/L65B-M1-C05_Llama65B_megatron_tp2_pp8_mbs1.csv @@ -0,0 +1,21074 @@ +[ pp_stage0: [rank0, rank1, rank2, rank3, rank4, rank5, rank6, rank7, rank8, rank9, rank10, rank11, rank12, rank13, rank14, rank15] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_next,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +[ pp_stage1: [rank16, rank17, rank18, rank19, rank20, rank21, rank22, rank23, rank24, rank25, rank26, rank27, rank28, rank29, rank30, rank31] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_next,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +[ pp_stage2: [rank32, rank33, rank34, rank35, rank36, rank37, rank38, rank39, rank40, rank41, rank42, rank43, rank44, rank45, rank46, rank47] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_next,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +[ pp_stage3: [rank48, rank49, rank50, rank51, rank52, rank53, rank54, rank55, rank56, rank57, rank58, rank59, rank60, rank61, rank62, rank63] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_next,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +[ pp_stage4: [rank64, rank65, rank66, rank67, rank68, rank69, rank70, rank71, rank72, rank73, rank74, rank75, rank76, rank77, rank78, rank79] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_next,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +[ pp_stage5: [rank80, rank81, rank82, rank83, rank84, rank85, rank86, rank87, rank88, rank89, rank90, rank91, rank92, rank93, rank94, rank95] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_next,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +[ pp_stage6: [rank96, rank97, rank98, rank99, rank100, rank101, rank102, rank103, rank104, rank105, rank106, rank107, rank108, rank109, rank110, rank111] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_next,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,forward_step,None,None,send_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,backward_step,None,None,recv_next,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +[ pp_stage7: [rank112, rank113, rank114, rank115, rank116, rank117, rank118, rank119, rank120, rank121, rank122, rank123, rank124, rank125, rank126, rank127] ] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.pp_group,8,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.embedding_group,2,262144000,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.embedding_group,2,33554432,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,8,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.irecv,CommGroup.pp_group,1,33554432,forward_step,None,None,recv_prev,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,33554432,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.isend,CommGroup.pp_group,1,33554432,backward_step,None,None,send_prev,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 diff --git a/aicb/workload/physical/model_workload/L7B-D1-C01_Llama7B_zero2_mbs1.csv b/aicb/workload/physical/model_workload/L7B-D1-C01_Llama7B_zero2_mbs1.csv new file mode 100644 index 000000000..0c5ed2e19 --- /dev/null +++ b/aicb/workload/physical/model_workload/L7B-D1-C01_Llama7B_zero2_mbs1.csv @@ -0,0 +1,310 @@ +[rank0 ~ rank127] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.broadcast,CommGroup.dp_group,128,262144000,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,262144000,init,None,None,,None,None,None,1 +CommType.barrier,CommGroup.all,128,262144000,init.__init__,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1971396608,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1933656064,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1990270976,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1990279168,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1990279168,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1990279168,backward.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1610670080,step.allreduce_bucket,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1,step.has_overflow,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2246138624,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2246138624,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2246138624,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2246138624,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2246138624,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,2246138624,step.all_gather_dp_groups,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 diff --git a/aicb/workload/physical/model_workload/L7B-D1-C02_Llama7B_zero3_mbs1.csv b/aicb/workload/physical/model_workload/L7B-D1-C02_Llama7B_zero3_mbs1.csv new file mode 100644 index 000000000..93d23cece --- /dev/null +++ b/aicb/workload/physical/model_workload/L7B-D1-C02_Llama7B_zero3_mbs1.csv @@ -0,0 +1,1245 @@ +[rank0 ~ rank127] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.broadcast,CommGroup.dp_group,128,262144000,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,33554432,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,90177536,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,8192,init._broadcast_model,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.dp_group,128,262144000,init._broadcast_model,None,0,,None,None,None,1 +CommType.barrier,CommGroup.all,128,0,init._create_fp16_partitions_with_defragmentation,None,None,,None,None,None,1 +CommType.barrier,CommGroup.all,128,0,init._setup_for_real_optimizer,None,None,,None,None,None,1 +CommType.barrier,CommGroup.all,128,0,init._setup_for_real_optimizer,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,262144000,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,init._allgather_params,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,262144000,init._allgather_params,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,262144000,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,262144000,forward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,262144000,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1971396608,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1933656064,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1990270976,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1990279168,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1990279168,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1990279168,backward.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,90177536,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,33554432,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,backward.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,262144000,backward.allgather_fn,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,128,1610670080,step.reduce_scatter_fn,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,1,step.has_overflow,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,128,8,step.get_grad_norm_direct,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,128,8192,step.allgather_fn,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 diff --git a/aicb/workload/physical/model_workload/L7B-M1-C04_Llama7B_megatron_tp2_pp1_mbs1.csv b/aicb/workload/physical/model_workload/L7B-M1-C04_Llama7B_megatron_tp2_pp1_mbs1.csv new file mode 100644 index 000000000..3bc9f2e98 --- /dev/null +++ b/aicb/workload/physical/model_workload/L7B-M1-C04_Llama7B_megatron_tp2_pp1_mbs1.csv @@ -0,0 +1,151 @@ +[rank0 ~ rank127] +comm_type,comm_group,comm_group_size,msg_size,stage,dst,src,additional,_elapsed_time,algbw,busbw,count +CommType.all_reduce,CommGroup.dp_group,64,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,64,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,64,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,64,8,init.model_setup,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,64,32,init.model_setup,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,24,init.model_setup,None,0,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,64,64,init.model_setup,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,40,forward_step,None,0,,None,None,None,1 +CommType.broadcast,CommGroup.tp_group,2,17408,forward_step,None,0,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,forward.MegatronRowLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,8192,forward_step._VocabParallelCrossEntropy,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.dp_group,64,4,forward_step.average_losses_across_data_parallel_group,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronColumnLinear,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,16777216,backward.MegatronEmbedding,None,None,,None,None,None,1 +CommType.reduce_scatter,CommGroup.dp_group,64,10366697472,step,None,None,,None,None,None,1 +CommType.all_gather,CommGroup.dp_group,64,5183348736,step,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,0,step._allreduce_layernorm_grads,None,None,,None,None,None,1 +CommType.all_reduce,CommGroup.tp_group,2,4,step.check_for_nan,None,None,,None,None,None,1 +CommType.epoch_end,None,None,0,,None,None,,None,None,None,1 diff --git a/aicb/workload/simAI/micro_test/all_gather.txt b/aicb/workload/simAI/micro_test/all_gather.txt new file mode 100644 index 000000000..2cdc63555 --- /dev/null +++ b/aicb/workload/simAI/micro_test/all_gather.txt @@ -0,0 +1,21 @@ +MICRO +19 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLGATHER 4096 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLGATHER 8192 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLGATHER 16384 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLGATHER 32768 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLGATHER 65536 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLGATHER 131072 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLGATHER 262144 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLGATHER 524288 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLGATHER 1048576 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLGATHER 2097152 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLGATHER 4194304 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLGATHER 8388608 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLGATHER 16777216 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLGATHER 33554432 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLGATHER 67108864 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLGATHER 134217728 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLGATHER 268435456 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLGATHER 536870912 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLGATHER 1073741824 1 diff --git a/aicb/workload/simAI/micro_test/all_reduce.txt b/aicb/workload/simAI/micro_test/all_reduce.txt new file mode 100644 index 000000000..a95106b8b --- /dev/null +++ b/aicb/workload/simAI/micro_test/all_reduce.txt @@ -0,0 +1,21 @@ +MICRO +19 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 4096 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 8192 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 16384 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 32768 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 65536 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 131072 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 262144 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 524288 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 1048576 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 2097152 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 4194304 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 8388608 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 16777216 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 33554432 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 67108864 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 134217728 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 268435456 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 536870912 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 1073741824 1 diff --git a/aicb/workload/simAI/micro_test/all_to_all.txt b/aicb/workload/simAI/micro_test/all_to_all.txt new file mode 100644 index 000000000..c36d1280d --- /dev/null +++ b/aicb/workload/simAI/micro_test/all_to_all.txt @@ -0,0 +1,21 @@ +MICRO +19 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLTOALL 4096 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLTOALL 8192 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLTOALL 16384 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLTOALL 32768 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLTOALL 65536 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLTOALL 131072 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLTOALL 262144 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLTOALL 524288 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLTOALL 1048576 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLTOALL 2097152 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLTOALL 4194304 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLTOALL 8388608 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLTOALL 16777216 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLTOALL 33554432 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLTOALL 67108864 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLTOALL 134217728 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLTOALL 268435456 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLTOALL 536870912 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLTOALL 1073741824 1 diff --git a/aicb/workload/simAI/micro_test/muti_all_reduce.txt b/aicb/workload/simAI/micro_test/muti_all_reduce.txt new file mode 100644 index 000000000..406e61f43 --- /dev/null +++ b/aicb/workload/simAI/micro_test/muti_all_reduce.txt @@ -0,0 +1,21 @@ +HYBRID_TRANSFORMER_FWD_IN_BCKWD model_parallel_NPU_group: 8 checkpoints: 0 checkpoint_initiates: 0 +19 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 4096 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 8192 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 16384 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 32768 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 65536 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 131072 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 262144 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 524288 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 1048576 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 2097152 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 4194304 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 8388608 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 16777216 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 33554432 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 67108864 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 134217728 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 268435456 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 536870912 1 +micro_test -1 1 NONE 0 1 NONE 0 1 ALLREDUCE 1073741824 1 diff --git a/aicb/workload/simAI/model_workload/G13B-M1-C01_GPT13B_megatron_tp8_pp1_mbs1_A100.txt b/aicb/workload/simAI/model_workload/G13B-M1-C01_GPT13B_megatron_tp8_pp1_mbs1_A100.txt new file mode 100644 index 000000000..f05fae824 --- /dev/null +++ b/aicb/workload/simAI/model_workload/G13B-M1-C01_GPT13B_megatron_tp8_pp1_mbs1_A100.txt @@ -0,0 +1,94 @@ +HYBRID_TRANSFORMER_FWD_IN_BCKWD model_parallel_NPU_group: 8 checkpoints: 0 checkpoint_initiates: 0 +92 +norm -1 0 BROADCAST 16384 1 NONE 0 1 NONE 0 100 +grad_norm -1 14565000 ALLGATHER 3248619520 13391000 NONE 0 1 REDUCESCATTER 6497239040 100 +layernorm -1 1 NONE 0 1 ALLREDUCE 3248619520 1 NONE 0 100 +embedding_layer -1 1 ALLREDUCE 20971520 1 ALLREDUCE 20971520 1 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +attention_layer -1 752000 ALLREDUCE 20971520 752000 ALLREDUCE 20971520 752000 NONE 0 100 +mlp_layer -1 750000 ALLREDUCE 20971520 750000 ALLREDUCE 20971520 750000 NONE 0 100 +embedding_norm -1 1 ALLREDUCE 327680000 1 NONE 0 1 NONE 0 100 +cross_entropy1 -1 1 ALLREDUCE 8192 1 NONE 0 1 NONE 0 100 +cross_entropy2 -1 1 ALLREDUCE 8192 1 NONE 0 1 NONE 0 100 +cross_entropy3 -1 1 ALLREDUCE 8192 1 NONE 0 1 NONE 0 100 +optimizer1 -1 1 ALLREDUCE 4 1 NONE 0 1 NONE 0 100 +optimizer2 -1 1 ALLREDUCE 4 1 NONE 0 1 NONE 0 100 +optimizer3 -1 1 ALLREDUCE 4 1 NONE 0 1 NONE 0 100 +optimizer4 -1 1 ALLREDUCE 4 1 NONE 0 1 NONE 0 100 diff --git a/aicb/workload/simAI/model_workload/G13B-M1-C02_GPT13B_megatron_tp8_pp1_mbs1_sp_A100.txt b/aicb/workload/simAI/model_workload/G13B-M1-C02_GPT13B_megatron_tp8_pp1_mbs1_sp_A100.txt new file mode 100644 index 000000000..82bf8f2e5 --- /dev/null +++ b/aicb/workload/simAI/model_workload/G13B-M1-C02_GPT13B_megatron_tp8_pp1_mbs1_sp_A100.txt @@ -0,0 +1,174 @@ +HYBRID_TRANSFORMER_FWD_IN_BCKWD model_parallel_NPU_group: 8 checkpoints: 0 checkpoint_initiates: 0 +172 +norm -1 0 BROADCAST 16384 1 NONE 0 1 NONE 0 100 +grad_norm -1 14565000 ALLGATHER 3248619520 13391000 NONE 0 1 REDUCESCATTER 6497239040 100 +layernorm -1 1 NONE 0 1 ALLREDUCE 3248619520 1 NONE 0 100 +embedding_layer -1 1 ALLREDUCE 20971520 1 ALLREDUCE 20971520 1 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +attention_column_layer -1 375000 ALLGATHER 20971520 375000 REDUCESCATTER 20971520 375000 NONE 0 100 +attention_row_layer -1 376000 REDUCESCATTER 20971520 376000 ALLGATHER 20971520 376000 NONE 0 100 +mlp_column_layer -1 923000 ALLGATHER 20971520 923000 REDUCESCATTER 20971520 923000 NONE 0 100 +mlp_row_layer -1 1125000 REDUCESCATTER 20971520 1125000 ALLGATHER 20971520 1125000 NONE 0 100 +embedding_norm -1 1 ALLREDUCE 327680000 1 NONE 0 1 NONE 0 100 +cross_entropy1 -1 1 ALLREDUCE 8192 1 NONE 0 1 NONE 0 100 +cross_entropy2 -1 1 ALLREDUCE 8192 1 NONE 0 1 NONE 0 100 +cross_entropy3 -1 1 ALLREDUCE 8192 1 NONE 0 1 NONE 0 100 +optimizer1 -1 1 ALLREDUCE 4 1 NONE 0 1 NONE 0 100 +optimizer2 -1 1 ALLREDUCE 4 1 NONE 0 1 NONE 0 100 +optimizer3 -1 1 ALLREDUCE 4 1 NONE 0 1 NONE 0 100 +optimizer4 -1 1 ALLREDUCE 4 1 NONE 0 1 NONE 0 100 diff --git a/aicb/workload/simAI/model_workload/G175B-M1-C03_GPT175B_megatron_tp8_pp1_mbs1_A100.txt b/aicb/workload/simAI/model_workload/G175B-M1-C03_GPT175B_megatron_tp8_pp1_mbs1_A100.txt new file mode 100644 index 000000000..47947652b --- /dev/null +++ b/aicb/workload/simAI/model_workload/G175B-M1-C03_GPT175B_megatron_tp8_pp1_mbs1_A100.txt @@ -0,0 +1,35 @@ +HYBRID_TRANSFORMER_FWD_IN_BCKWD model_parallel_NPU_group: 8 checkpoints: 0 checkpoint_initiates: 0 +33 +grad_gather -1 1 NONE 0 13848000 NONE 0 1 ALLGATHER 1 100 +grad_param -1 1 NONE 0 13740000 NONE 0 1 REDUCESCATTER 11175321600 100 +layernorm -1 1 NONE 0 1 ALLREDUCE 5587660800 1 NONE 0 100 +embedding_grads -1 1 NONE 0 1 ALLREDUCE 150994944 1 NONE 0 100 +embedding_layer -1 523895 ALLREDUCE 150994944 1 NONE 0 1 NONE 0 100 +attention_norm -1 2571000 ALLREDUCE 50331648 2454000 NONE 0 2454000 NONE 0 100 +mlp_norm -1 1584000 ALLREDUCE 50331648 2081000 NONE 0 2081000 NONE 0 100 +attention_norm -1 2571000 ALLREDUCE 50331648 2454000 NONE 0 2454000 NONE 0 100 +mlp_norm -1 1584000 ALLREDUCE 50331648 2081000 NONE 0 2081000 NONE 0 100 +attention_norm -1 2571000 ALLREDUCE 50331648 2454000 NONE 0 2454000 NONE 0 100 +mlp_norm -1 1584000 ALLREDUCE 50331648 2081000 NONE 0 2081000 NONE 0 100 +attention_norm -1 2571000 ALLREDUCE 50331648 2454000 NONE 0 2454000 NONE 0 100 +mlp_norm -1 1584000 ALLREDUCE 50331648 2081000 NONE 0 2081000 NONE 0 100 +attention_norm -1 2571000 ALLREDUCE 50331648 2454000 NONE 0 2454000 NONE 0 100 +mlp_norm -1 1584000 ALLREDUCE 50331648 2081000 NONE 0 2081000 NONE 0 100 +attention_norm -1 2571000 ALLREDUCE 50331648 2454000 NONE 0 2454000 NONE 0 100 +mlp_norm -1 1584000 ALLREDUCE 50331648 2081000 NONE 0 2081000 NONE 0 100 +attention_norm -1 2571000 ALLREDUCE 50331648 2454000 NONE 0 2454000 NONE 0 100 +mlp_norm -1 1584000 ALLREDUCE 50331648 2081000 NONE 0 2081000 NONE 0 100 +attention_norm -1 2571000 ALLREDUCE 50331648 2454000 NONE 0 2454000 NONE 0 100 +mlp_norm -1 1584000 ALLREDUCE 50331648 2081000 NONE 0 2081000 NONE 0 100 +attention_norm -1 2571000 ALLREDUCE 50331648 2454000 NONE 0 2454000 NONE 0 100 +mlp_norm -1 1584000 ALLREDUCE 50331648 2081000 NONE 0 2081000 NONE 0 100 +attention_norm -1 2571000 ALLREDUCE 50331648 2454000 NONE 0 2454000 NONE 0 100 +mlp_norm -1 1584000 ALLREDUCE 50331648 2081000 NONE 0 2081000 NONE 0 100 +attention_norm -1 2571000 ALLREDUCE 50331648 2454000 NONE 0 2454000 NONE 0 100 +mlp_norm -1 1584000 ALLREDUCE 50331648 2081000 NONE 0 2081000 NONE 0 100 +attention_norm -1 2571000 ALLREDUCE 50331648 2454000 NONE 0 2454000 NONE 0 100 +mlp_norm -1 1584000 ALLREDUCE 50331648 2081000 NONE 0 2081000 NONE 0 100 +embedding_norm -1 1 ALLREDUCE 786432000 1 NONE 0 1 NONE 0 100 +cross_entropy1 -1 1 ALLREDUCE 8192 1 NONE 0 1 NONE 0 100 +cross_entropy2 -1 1 ALLREDUCE 8192 1 NONE 0 1 NONE 0 100 +cross_entropy3 -1 1 ALLREDUCE 8192 1 NONE 0 1 NONE 0 100 \ No newline at end of file diff --git a/aicb/workload/simAI/model_workload/L65B-M1-C05_Llama65B_megatron_tp8_pp1_mbs1_A100.txt b/aicb/workload/simAI/model_workload/L65B-M1-C05_Llama65B_megatron_tp8_pp1_mbs1_A100.txt new file mode 100644 index 000000000..b03d37acd --- /dev/null +++ b/aicb/workload/simAI/model_workload/L65B-M1-C05_Llama65B_megatron_tp8_pp1_mbs1_A100.txt @@ -0,0 +1,35 @@ +HYBRID_TRANSFORMER_FWD_IN_BCKWD model_parallel_NPU_group: 8 checkpoints: 0 checkpoint_initiates: 0 +839 +grad_gather -1 1 NONE 0 4284000 NONE 0 1 ALLGATHER 1 100 +grad_param -1 1 NONE 0 7300000 NONE 0 1 REDUCESCATTER 6236471296 100 +layernorm -1 1 NONE 0 1 ALLREDUCE 2315808768 1 NONE 0 100 +embedding_grads -1 1 NONE 0 1 ALLREDUCE 65536000 1 NONE 0 100 +embedding_layer -1 382000 ALLREDUCE 65536000 1 NONE 0 1 NONE 0 100 +attention_norm -1 1711000 ALLREDUCE 33554432 1500000 NONE 0 1500000 NONE 0 100 +mlp_norm -1 862000 ALLREDUCE 33554432 1200000 NONE 0 1200000 NONE 0 100 +attention_norm -1 1711000 ALLREDUCE 33554432 1500000 NONE 0 1500000 NONE 0 100 +mlp_norm -1 862000 ALLREDUCE 33554432 1200000 NONE 0 1200000 NONE 0 100 +attention_norm -1 1711000 ALLREDUCE 33554432 1500000 NONE 0 1500000 NONE 0 100 +mlp_norm -1 862000 ALLREDUCE 33554432 1200000 NONE 0 1200000 NONE 0 100 +attention_norm -1 1711000 ALLREDUCE 33554432 1500000 NONE 0 1500000 NONE 0 100 +mlp_norm -1 862000 ALLREDUCE 33554432 1200000 NONE 0 1200000 NONE 0 100 +attention_norm -1 1711000 ALLREDUCE 33554432 1500000 NONE 0 1500000 NONE 0 100 +mlp_norm -1 862000 ALLREDUCE 33554432 1200000 NONE 0 1200000 NONE 0 100 +attention_norm -1 1711000 ALLREDUCE 33554432 1500000 NONE 0 1500000 NONE 0 100 +mlp_norm -1 862000 ALLREDUCE 33554432 1200000 NONE 0 1200000 NONE 0 100 +attention_norm -1 1711000 ALLREDUCE 33554432 1500000 NONE 0 1500000 NONE 0 100 +mlp_norm -1 862000 ALLREDUCE 33554432 1200000 NONE 0 1200000 NONE 0 100 +attention_norm -1 1711000 ALLREDUCE 33554432 1500000 NONE 0 1500000 NONE 0 100 +mlp_norm -1 862000 ALLREDUCE 33554432 1200000 NONE 0 1200000 NONE 0 100 +attention_norm -1 1711000 ALLREDUCE 33554432 1500000 NONE 0 1500000 NONE 0 100 +mlp_norm -1 862000 ALLREDUCE 33554432 1200000 NONE 0 1200000 NONE 0 100 +attention_norm -1 1711000 ALLREDUCE 33554432 1500000 NONE 0 1500000 NONE 0 100 +mlp_norm -1 862000 ALLREDUCE 33554432 1200000 NONE 0 1200000 NONE 0 100 +attention_norm -1 1711000 ALLREDUCE 33554432 1500000 NONE 0 1500000 NONE 0 100 +mlp_norm -1 862000 ALLREDUCE 33554432 1200000 NONE 0 1200000 NONE 0 100 +attention_norm -1 1711000 ALLREDUCE 33554432 1500000 NONE 0 1500000 NONE 0 100 +mlp_norm -1 862000 ALLREDUCE 33554432 1200000 NONE 0 1200000 NONE 0 100 +embedding_norm -1 1 ALLREDUCE 524288000 1 NONE 0 1 NONE 0 100 +cross_entropy1 -1 1 ALLREDUCE 8192 1 NONE 0 1 NONE 0 100 +cross_entropy2 -1 1 ALLREDUCE 8192 1 NONE 0 1 NONE 0 100 +cross_entropy3 -1 1 ALLREDUCE 8192 1 NONE 0 1 NONE 0 100 \ No newline at end of file diff --git a/aicb/workload/simAI/model_workload/L65B_D1_C08_Llama65B_deepspeed_zero3_A100.txt b/aicb/workload/simAI/model_workload/L65B_D1_C08_Llama65B_deepspeed_zero3_A100.txt new file mode 100644 index 000000000..7de93476a --- /dev/null +++ b/aicb/workload/simAI/model_workload/L65B_D1_C08_Llama65B_deepspeed_zero3_A100.txt @@ -0,0 +1,14 @@ +HYBRID_TRANSFORMER_FWD_IN_BCKWD model_parallel_NPU_group: 1 checkpoints: 0 checkpoint_initiates: 0 +12 +Llama_layer1 -1 14356000 NONE 0 4756000 ALLGATHER 357468 1 REDUCESCATTER 1889785610 100 +Llama_layer2 -1 10948000 NONE 0 19502000 ALLGATHER 119162 1 NONE 0 100 +Llama_layer3 -1 7168000 NONE 0 13844000 ALLGATHER 317843 1 NONE 0 100 +Llama_layer4 -1 17976000 NONE 0 14664000 ALLGATHER 317843 1 NONE 0 100 +Llama_layer5 -1 14752000 NONE 0 4607000 ALLGATHER 317843 1 NONE 0 100 +Llama_layer6 -1 8000000 NONE 0 46148000 ALLGATHER 357468 1 NONE 0 100 +Llama_layer7 -1 7168000 NONE 0 20196000 ALLGATHER 119162 1 NONE 0 100 +Llama_layer8 -1 1760000 NONE 0 13844000 ALLGATHER 317843 1 NONE 0 100 +Llama_layer9 -1 15236000 NONE 0 15236000 ALLGATHER 317843 1 NONE 0 100 +Llama_layer10 -1 7008000 NONE 0 16036000 ALLGATHER 317843 1 NONE 0 100 +Llama_layer11 -1 16036000 NONE 0 4607000 ALLGATHER 317843 1 NONE 0 100 +Llama_layer12 -1 7064500 NONE 0 21750000 ALLGATHER 317843 1 NONE 0 100 \ No newline at end of file diff --git a/aicb/workload/simAI/model_workload/L7B-D1-C02_Llama7B_deepspeed_zero3_A100.txt b/aicb/workload/simAI/model_workload/L7B-D1-C02_Llama7B_deepspeed_zero3_A100.txt new file mode 100644 index 000000000..7de93476a --- /dev/null +++ b/aicb/workload/simAI/model_workload/L7B-D1-C02_Llama7B_deepspeed_zero3_A100.txt @@ -0,0 +1,14 @@ +HYBRID_TRANSFORMER_FWD_IN_BCKWD model_parallel_NPU_group: 1 checkpoints: 0 checkpoint_initiates: 0 +12 +Llama_layer1 -1 14356000 NONE 0 4756000 ALLGATHER 357468 1 REDUCESCATTER 1889785610 100 +Llama_layer2 -1 10948000 NONE 0 19502000 ALLGATHER 119162 1 NONE 0 100 +Llama_layer3 -1 7168000 NONE 0 13844000 ALLGATHER 317843 1 NONE 0 100 +Llama_layer4 -1 17976000 NONE 0 14664000 ALLGATHER 317843 1 NONE 0 100 +Llama_layer5 -1 14752000 NONE 0 4607000 ALLGATHER 317843 1 NONE 0 100 +Llama_layer6 -1 8000000 NONE 0 46148000 ALLGATHER 357468 1 NONE 0 100 +Llama_layer7 -1 7168000 NONE 0 20196000 ALLGATHER 119162 1 NONE 0 100 +Llama_layer8 -1 1760000 NONE 0 13844000 ALLGATHER 317843 1 NONE 0 100 +Llama_layer9 -1 15236000 NONE 0 15236000 ALLGATHER 317843 1 NONE 0 100 +Llama_layer10 -1 7008000 NONE 0 16036000 ALLGATHER 317843 1 NONE 0 100 +Llama_layer11 -1 16036000 NONE 0 4607000 ALLGATHER 317843 1 NONE 0 100 +Llama_layer12 -1 7064500 NONE 0 21750000 ALLGATHER 317843 1 NONE 0 100 \ No newline at end of file diff --git a/aicb/workload/simAI/model_workload/L7B-M1-C04_Llama7B_megatron_tp2_pp1_mbs1_A100.txt b/aicb/workload/simAI/model_workload/L7B-M1-C04_Llama7B_megatron_tp2_pp1_mbs1_A100.txt new file mode 100644 index 000000000..4f46c5b53 --- /dev/null +++ b/aicb/workload/simAI/model_workload/L7B-M1-C04_Llama7B_megatron_tp2_pp1_mbs1_A100.txt @@ -0,0 +1,78 @@ +HYBRID_TRANSFORMER_FWD_IN_BCKWD model_parallel_NPU_group: 2 checkpoints: 0 checkpoint_initiates: 0 +76 +norm -1 0 BROADCAST 16384 1 NONE 0 1 NONE 0 100 +grad_norm -1 1 ALLGATHER 6754926592 1 NONE 0 1 REDUCESCATTER 13509853184 100 +layernorm -1 1 NONE 0 1 ALLREDUCE 6754926592 1 NONE 0 100 +embedding_layer -1 1 ALLREDUCE 16777216 1 ALLREDUCE 16777216 1 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +attention_layer -1 539986 ALLREDUCE 16777216 592000 ALLREDUCE 16777216 592000 NONE 0 100 +mlp_layer -1 486330 ALLREDUCE 16777216 529500 ALLREDUCE 16777216 529500 NONE 0 100 +embedding_norm -1 1 ALLREDUCE 262144000 1 NONE 0 1 NONE 0 100 +cross_entropy1 -1 1 ALLREDUCE 8192 1 NONE 0 1 NONE 0 100 +cross_entropy2 -1 1 ALLREDUCE 8192 1 NONE 0 1 NONE 0 100 +cross_entropy3 -1 1 ALLREDUCE 8192 1 NONE 0 1 NONE 0 100 +optimizer1 -1 1 ALLREDUCE 4 1 NONE 0 1 NONE 0 100 +optimizer2 -1 1 ALLREDUCE 4 1 NONE 0 1 NONE 0 100 +optimizer3 -1 1 ALLREDUCE 4 1 NONE 0 1 NONE 0 100 +optimizer4 -1 1 ALLREDUCE 4 1 NONE 0 1 NONE 0 100 diff --git a/aicb/workload_applyer.py b/aicb/workload_applyer.py new file mode 100755 index 000000000..34b4dca4c --- /dev/null +++ b/aicb/workload_applyer.py @@ -0,0 +1,397 @@ +""" +Copyright (c) 2021, Alibaba Group; +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import torch +import sys +import math +import time +from utils.utils import WorkloadWriter, CommGroup, CommType, ReduceOp +from utils.benchmark_logger import bench_logger +import utils.utils as utils + + +class WorkloadApplyer: + def __init__(self, workload=None, args=None, filename=None) -> None: + if workload is None or args is None: + assert ( + filename is None + ), f"you should either pass workload,args or filename to init WorkloadApplyer" + workload, args = WorkloadWriter.load_workload(filename) + # if not hasattr(args, "backend"): + # args.backend = "nccl" + # torch.distributed.init_process_group(backend=args.backend) + self.args = args + world_size = torch.distributed.get_world_size() + # args.rank = torch.distributed.get_rank() + if args.world_size != world_size: + print( + f"WARNNING: world_size is {args.world_size} when generating workload, but now world size is {world_size}" + ) + args.world_size = torch.distributed.get_world_size() + device_count = torch.cuda.device_count() + self.device = args.rank % device_count + torch.cuda.set_device(self.device) + self.device = torch.cuda.current_device() + self.comm_group_info, self.pp_global_rank_info = ( + self._generate_dp_tp_pp_ep_groups() + ) + self.workload = workload + self.comm_type_function = { + CommType.barrier: self._apply_barrier, + CommType.broadcast: self._apply_broadcast, + CommType.reduce: self._apply_reduce, + CommType.all_reduce: self._apply_all_reduce, + CommType.all_gather: self._apply_all_gather, + CommType.reduce_scatter: self._apply_reduce_scatter, + CommType.isend: self._apply_p2pcommunication, + CommType.irecv: self._apply_p2pcommunication, + CommType.all_gather_into_tensor: self._apply_all_gather, + CommType.reduce_scatter_tensor: self._apply_reduce_scatter, + CommType.computation: self._apply_computation, + CommType.all_to_all: self._apply_all_to_all, + CommType.epoch_end: bench_logger.end_epoch, + + } + + cal_tuple_num = lambda t: math.prod(t[0]) + math.prod(t[1]) + max_msg_size = max( + [ + ( + item.msg_size + if isinstance(item.msg_size, int) + else cal_tuple_num(item.msg_size) + ) + for item in self.workload.workload + ] + ) + self.gemm_cache = {} + self.computation_aiob = False + if args.aiob_enable and (args.frame == "Megatron" or args.frame == "DeepSeek"): + self.computation_aiob = True + + self.skip_computation = False + self.always_apply_gemm = False + self.gemm_iters = 1 if self.always_apply_gemm else 50 + self.buffer = torch.empty( + (max_msg_size,), dtype=torch.bfloat16, device=self.device + ) + def _generate_dp_tp_pp_ep_groups(self): + """Borrow from Megatron-LM""" + all_data_parallel_group_ranks = [] + world_size = self.args.world_size + rank = torch.distributed.get_rank() + self.rank = rank + tensor_model_parallel_size, pipeline_model_parallel_size, data_parallel_size,expert_model_parallel_size = ( + self.args.tensor_model_parallel_size, + self.args.pipeline_model_parallel, + self.args.dp_num, + self.args.expert_model_parallel_size, + ) + rank_generator = utils.RankGenerator( + tp=tensor_model_parallel_size, + ep=expert_model_parallel_size, + dp=data_parallel_size, + pp=pipeline_model_parallel_size, + cp=self.args.context_parallel_size, + order='tp-cp-ep-dp-pp', + ) + for ranks in rank_generator.get_ranks('ep', independent_ep=True): + group = torch.distributed.new_group( + ranks + ) + if rank in ranks: + ep_group = group + for ranks in rank_generator.get_ranks('tp'): + group = torch.distributed.new_group( + ranks + ) + if rank in ranks: + tp_group = group + for ranks in rank_generator.get_ranks('pp'): + group = torch.distributed.new_group( + ranks + ) + if rank in ranks: + pp_group = group + pp_global_rank = ranks + # Setup embedding group (to exchange gradients between + # first and last stages). + # if len(ranks) > 1: + # embedding_ranks = [ranks[0], ranks[-1]] + # position_embedding_ranks = [ranks[0]] + # if self.args.pipeline_model_parallel_split_rank is not None: + # if ranks[self.args.pipeline_model_parallel_split_rank] not in embedding_ranks: + # embedding_ranks = [ + # ranks[0], + # ranks[self.args.pipeline_model_parallel_split_rank], + # ranks[-1], + # ] + # if ranks[self.args.pipeline_model_parallel_split_rank] not in position_embedding_ranks: + # position_embedding_ranks = [ranks[0], ranks[self.args.pipeline_model_parallel_split_rank]] + # else: + # embedding_ranks = ranks + # position_embedding_ranks = ranks + + # group = torch.distributed.new_group( + # embedding_ranks + # ) + # if rank in embedding_ranks: + # _EMBEDDING_GROUP = group + # if rank in ranks: + # _EMBEDDING_GLOBAL_RANKS = embedding_ranks + + # group = torch.distributed.new_group( + # position_embedding_ranks, + + # ) + # if rank in position_embedding_ranks: + # _POSITION_EMBEDDING_GROUP = group + # if rank in ranks: + # _POSITION_EMBEDDING_GLOBAL_RANKS = position_embedding_ranks + for ranks in rank_generator.get_ranks('dp'): + group = torch.distributed.new_group( + ranks + ) + if rank in ranks: + dp_group = group + for ranks in rank_generator.get_ranks('tp-ep', independent_ep=True): + group = torch.distributed.new_group( + ranks + ) + if rank in ranks: + ep_tp_group = group + for ranks in rank_generator.get_ranks('dp', independent_ep=True): + group = torch.distributed.new_group( + ranks + ) + if rank in ranks: + ep_dp_group = group + return { + CommGroup.tp_group: tp_group, + CommGroup.dp_group: dp_group, + CommGroup.pp_group: pp_group, + CommGroup.ep_group: ep_group, + CommGroup.ep_tp_group: ep_tp_group, + CommGroup.ep_dp_group: ep_dp_group, + }, pp_global_rank + + def _get_pipeline_parallel_size(self): + group = self.comm_group_info["pp_group"] + pp_group_size = torch.distributed.get_world_size(group) + return pp_group_size + + def _get_pipeline_parallel_rank(self): + group = self.comm_group_info["pp_group"] + pp_rank = torch.distributed.get_rank(group) + return pp_rank + + def _get_pipeline_prev_rank(self): + rank_in_pipeline = self._get_pipeline_parallel_rank() + world_size = self._get_pipeline_parallel_size() + return self.pp_global_rank_info[(rank_in_pipeline - 1) % world_size] + + def _get_pipeline_next_rank(self): + rank_in_pipeline = self._get_pipeline_parallel_rank() + world_size = self._get_pipeline_parallel_size() + return self.pp_global_rank_info[(rank_in_pipeline + 1) % world_size] + + @bench_logger.log_timing("comm") + def _apply_p2pcommunication(self, item): + ops = [] + tensor = torch.narrow(self.buffer, 0, 0, item.msg_size // 2) + if item.additional == "send_prev": + if self._get_pipeline_parallel_rank() != 0: + send_prev_op = torch.distributed.P2POp( + torch.distributed.isend, tensor, self._get_pipeline_prev_rank() + ) + ops.append(send_prev_op) + else: + pass + if item.additional == "send_next": + if self._get_pipeline_parallel_rank() != self.args.pipeline_model_parallel - 1: + send_next_op = torch.distributed.P2POp( + torch.distributed.isend, tensor, self._get_pipeline_next_rank() + ) + ops.append(send_next_op) + else: + pass + if item.additional == "recv_prev": + if self._get_pipeline_parallel_rank() != 0: + tensor_recv_prev = torch.empty( + item.msg_size // 2, dtype=torch.bfloat16, device=self.device + ) + recv_prev_op = torch.distributed.P2POp( + torch.distributed.irecv, + tensor_recv_prev, + self._get_pipeline_prev_rank(), + ) + ops.append(recv_prev_op) + else: + pass + if item.additional == "recv_next": + if self._get_pipeline_parallel_rank() != self.args.pipeline_model_parallel - 1: + tensor_recv_next = torch.empty( + item.msg_size // 2, dtype=torch.bfloat16, device=self.device + ) + recv_next_op = torch.distributed.P2POp( + torch.distributed.irecv, + tensor_recv_next, + self._get_pipeline_next_rank(), + ) + ops.append(recv_next_op) + else: + pass + if len(ops) > 0: + reqs = torch.distributed.batch_isend_irecv(ops) + for req in reqs: + req.wait() + + torch.cuda.synchronize() + + def _apply_barrier(self, item): + torch.distributed.barrier() + + @bench_logger.log_timing("comm") + def _apply_broadcast(self, item): + tensor = torch.narrow(self.buffer, 0, 0, item.msg_size // 2) + group = self.comm_group_info[item.comm_group] + src = torch.distributed.get_global_rank(group, 0) + return torch.distributed.broadcast( + tensor=tensor, src=src, group=group, async_op=False + ) + + @bench_logger.log_timing("comm") + def _apply_reduce(self, item): + tensor = torch.narrow(self.buffer, 0, 0, item.msg_size // 2) + group = self.comm_group_info[item.comm_group] + dst = item.dst + return torch.distributed.reduce( + tensor=tensor, + dst=dst, + op=torch.distributed.ReduceOp.SUM, + group=group, + async_op=False, + ) + + @bench_logger.log_timing("comm") + def _apply_all_reduce(self, item): + tensor = torch.narrow(self.buffer, 0, 0, item.msg_size // 2) + group = self.comm_group_info[item.comm_group] + return torch.distributed.all_reduce( + tensor=tensor, + op=torch.distributed.ReduceOp.SUM, + group=group, + async_op=False, + ) + + @bench_logger.log_timing("comm") + def _apply_all_gather(self, item): + group = self.comm_group_info[item.comm_group] + num_elements = item.msg_size // 2 + padding_size = ( + (group.size() - num_elements % group.size()) + if num_elements % group.size() + else 0 + ) + num_elements = num_elements + padding_size + output_tensor = torch.narrow(self.buffer, 0, 0, num_elements) + input_tensor_size = output_tensor.numel() // group.size() + group_rank = torch.distributed.get_group_rank(group, self.rank) + input_tensor = torch.narrow( + output_tensor, 0, group_rank * input_tensor_size, input_tensor_size + ) + return torch.distributed.all_gather_into_tensor( + output_tensor, input_tensor, group=group, async_op=False + ) + @bench_logger.log_timing("comm") + def _overlap(self, item): + item.additional = 'overlap' + + @bench_logger.log_timing("comm") + def _apply_reduce_scatter(self, item): + group = self.comm_group_info[item.comm_group] + num_elements = item.msg_size // 2 + padding_size = ( + (group.size() - num_elements % group.size()) + if num_elements % group.size() + else 0 + ) + num_elements = num_elements + padding_size + input_tensor = torch.narrow(self.buffer, 0, 0, num_elements) + group = self.comm_group_info[item.comm_group] + output_tensor_size = input_tensor.numel() // group.size() + group_rank = torch.distributed.get_group_rank(group, self.rank) + output_tensor = torch.narrow( + input_tensor, 0, group_rank * output_tensor_size, output_tensor_size + ) + return torch.distributed.reduce_scatter_tensor( + output_tensor, input_tensor, group=group, async_op=False + ) + + @bench_logger.log_timing("comm") + def _apply_all_to_all(self, item): + group = self.comm_group_info[item.comm_group] + num_elements = item.msg_size // 2 + input_tensor = torch.narrow(self.buffer, 0, 0, num_elements) + # output_tensor = torch.narrow(self.buffer, 0, 0 , num_elements) + output_tensor = torch.empty( + num_elements * group.size(), + dtype=self.buffer.dtype, + device=self.buffer.device, + ) + return torch.distributed.all_to_all_single( + output_tensor, input_tensor, group=group + ) + + @bench_logger.log_timing("comp") + def _apply_computation(self, item): + if self.skip_computation: + return + if self.computation_aiob: + time.sleep(item._elapsed_time/ 1e9) + else: + # item.msg_size = 1 + input_shape1, input_shape2 = item.msg_size + A, B = torch.rand(input_shape1, device=self.device), torch.rand( + input_shape2, device=self.device + ) + torch.matmul(A, B) + return + + def apply_workload(self): + torch.cuda.synchronize(self.device) + start = time.perf_counter() + key = "backward" + for item in self.workload.workload: + if ( + self.computation_aiob + and item.comm_type == CommType.all_reduce + and key in item.stage + ): + comm_func = self.comm_type_function[item.comm_type] + # comm_func = self._overlap() + # comm_func(item) + else: + comm_func = self.comm_type_function[item.comm_type] + comm_func(item) + torch.cuda.synchronize(self.device) + end = time.perf_counter() + return end - start + + +if __name__ == "__main__": + filename = "results/model_workload/local_deepspeed_stage3.csv" + applyer = WorkloadApplyer(filename=filename) + applyer.apply_workload() + # timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S") + if torch.distributed.get_rank() == 0: + bench_logger.analyze_comm_log(bench_logger.comm_log) diff --git a/aicb/workload_generator/CLAUDE.md b/aicb/workload_generator/CLAUDE.md new file mode 100644 index 000000000..478ca44ad --- /dev/null +++ b/aicb/workload_generator/CLAUDE.md @@ -0,0 +1,481 @@ +# AICB Workload Generator — CLAUDE.md + +> **Research reference**: `RESEARCH.md` contains the definitive architecture +> comparison (LLaMA 3 vs Qwen3 vs Qwen3.5), verified model parameter tables +> for all 22 supported models, and the complete research methodology (28 sources +> across 5 sub-questions). This CLAUDE.md is the operational reference; consult +> RESEARCH.md for the "why" behind the architecture decisions. +> +> Primary external sources: +> - Qwen3 Technical Report: ar5iv.labs.arxiv.org/html/2505.09388 (140KB HTML) +> - Qwen3.5 official blog: qwen-ai.com/qwen-3-5 (38KB, March 2026) +> - Model configs: hf-mirror.com (16 models), modelscope.cn (1 gated model) +> - HF Transformers source: ghproxy.net (modeling_qwen3.py, modeling_qwen3_5.py, +> modeling_llama.py, modular_qwen3.py) + +## Architecture + +The AICB workload generator has two independent layers that share model configs: + +``` +Layer 1: Workload Mock (training/*.py, inference/*.py) + └── Declarative communication trace: builds a list of LogItem objects + describing TP/EP/DP/PP collective operations. No tensors allocated, + no GPUs touched. Output: CSV consumed by the SimAI simulator. + +Layer 2: AIOB Compute Benchmark (Aiob*.py) + └── Actual CUDA kernel timing: runs real PyTorch GEMMs, attention ops, + MoE routing on GPU and records microsecond wall-clock times. + Output: timing maps merged into the CSV for compute-aware simulation. +``` + +Both layers are dispatched by `generate_megatron_workload.py` based on `--frame`. + +### Entry Point + +``` +python -m workload_generator.generate_megatron_workload \ + --frame \ + [--config config.json] [--hidden_size 4096 ...] +``` + +Flow: `get_params()` parsers CLI args → `MegatronWorkload(args, model)` calls +`model.forward()` and `model.backward()` recursively → produces `Workload` object +→ dumped as CSV. + +The `WorkloadGenerator` base class (`workload_generator.py`) handles rank mapping, +pipeline parallelism scheduling, and optimizer step collectives. + +### How the Mock Works + +Each module (attention, MLP, MoE) implements `forward()` and `backward()` that +return a `Workload` object (list of `LogItem`). A `LogItem` is declarative: + +```python +LogItem( + comm_type=CommType.all_gather, # what kind of communication + comm_group=CommGroup.tp_group, # which process group + comm_group_size=8, # how many ranks + msg_size=67108864, # bytes transferred + stage="forward.Qwen3ColumnLinear" +) +``` + +Modules compose recursively: `Qwen3Model.forward()` calls `embedding.forward()`, +then each `layer.forward()`, then `lm_head.forward()`. Each layer's `forward()` +calls `attention.forward()` then `mlp.forward()`. The result is a flat list of +all communication events across the entire forward/backward pass. + +**Critical rule: the mock ONLY models events that cross GPU boundaries.** +Pure local compute (RMSNorm, SiLU, softmax, RoPE, QK-Norm, GatedDeltaNet +recurrence) produces ZERO LogItems. Only TP/EP/DP/PP collectives generate +entries. GatedDeltaNet layers return empty Workload -- all projections are +replicated (not TP-sharded); parameters are tracked via raw MockedParam for +DP gradient sync sizing. + +--- + +## Supported Model Families + +### Training Workload Mocks + +| Frame Name | File | Status | Models | +|---|---|---|---| +| `Megatron` | `training/MockedMegatron.py` | COMPLETE | LLaMA 3.1 8B/70B/405B | +| `DeepSeek` | `training/MockedDeepSeek.py` | COMPLETE | DeepSeek V3 (671B MoE), V3.1 | +| `Qwen3` | `training/MockedQwen3.py` | COMPLETE | Dense 0.6B-32B, MoE 30B-A3B, 235B-A22B | +| `Qwen3.5` | `training/MockedQwen3_5.py` | COMPLETE | Dense 0.8B-27B, MoE 35B-A3B, 122B-A10B, 397B-A17B | + +### Inference Workload Mocks + +| File | Status | +|---|---| +| `inference/MockedDeepSeek.py` | COMPLETE | +| `inference/MockedQwen3Moe.py` | SKELETAL (no forward/backward) | +| `inference/MockedQwen3Next.py` | SKELETAL (no forward/backward) | + +### AIOB Compute Benchmarks + +| File | Status | +|---|---| +| `training/AiobMegatron.py` (1107L) | COMPLETE | +| `training/AiobDeepSeek.py` (917L) | COMPLETE | +| `inference/AiobDeepSeek.py` (495L) | COMPLETE | +| `inference/AiobQwen3Moe.py` (427L) | COMPLETE | +| `inference/AiobQwen3Next.py` (697L) | COMPLETE | + +--- + +## Key Design Patterns + +### The ColumnLinear / RowLinear Comm Abstraction + +All TP communication is encapsulated in two dimension-agnostic classes: + +- **ColumnLinear**: shards output across TP. Forward = all-gather input, then + local matmul. Backward = local grad matmul, then reduce-scatter grad. +- **RowLinear**: shards input across TP. Forward = local matmul, then + reduce-scatter output. Backward = all-gather grad, then local matmul. + +These classes accept arbitrary `input_size` and `output_size` — the +communication message sizes are computed from `seq_len`, `batch_size`, and +`input_size` (for ColumnLinear all-gather) or `output_size` (for RowLinear +reduce-scatter). This makes them reusable across all architectures with zero +modification. + +### MoE Communication Pattern + +The `MOEMLP` class models the standard Megatron MoE pattern used by all +supported MoE models (Qwen3, Qwen3.5, DeepSeek, Megatron): + +``` +Forward: + 1. Shared expert MLP (if present): ColumnLinear all-gather + RowLinear reduce-scatter + 2. EP all-to-all dispatch: ship tokens to expert-owning ranks + 3. TP all-gather: gather full token batch within TP for grouped GEMM + 4. [expert FFN computation — not modeled] + 5. TP reduce-scatter: reduce partial expert outputs across TP + 6. EP all-to-all combine: ship results back to original token ranks + +Backward: same operations in reverse with gradient data +``` + +Message size formulas (all divide by ep_size after dispatch, fixed 2025-06-15): +- EP dispatch: `seq_len * hidden_size * batch_size * topk / tp / ep * 2` bytes +- TP all-gather: `2 * hidden_size * topk * batch_size * seq_len / ep` bytes +- TP reduce-scatter: same as TP all-gather +- EP combine: same as EP dispatch + +### Why Qwen3/Qwen3.5 Don't Need New MoE Classes + +- **Qwen3 MoE**: 128 experts, top-8, NO shared experts. Uses stock `MOEMLP` + directly — zero modifications. (Qwen3 Technical Report explicitly removed + shared experts from Qwen2.5-MoE.) + +- **Qwen3.5 MoE**: 256-512 experts, top-8/10, WITH shared experts (1 + always-active dense MLP). Uses `Qwen3_5MoEMLP` which extends the `MOEMLP` + pattern with a `shared_expert` MegatronMlp instance. + +### Qwen3 Attention vs Megatron Attention + +The only code difference is in `__init__` dimension computation: + +``` +MegatronAttention: + kv_channels = hidden_size // num_attention_heads + query_projection = kv_channels * num_attention_heads # = hidden_size + kv_projection = kv_channels * num_attention_heads # = hidden_size + qkv_output = 3 * hidden_size + +Qwen3Attention: + head_dim = 128 (from config, fixed across all models) + query_projection = head_dim * num_attention_heads # may ≠ hidden_size! + kv_projection = head_dim * num_key_value_heads # GQA-aware! + qkv_output = query_projection + 2 * kv_projection +``` + +`forward()` and `backward()` are byte-for-byte identical to MegatronAttention. +This correctly handles models where `num_heads * head_dim != hidden_size` +(e.g., Qwen3-4B: 32×128=4096 ≠ hidden=2560; Qwen3-0.6B: 16×128=2048 ≠ 1024). + +### Qwen3.5 GatedDeltaNet: Communication-Accurate Mock + +GatedDeltaNet is a linear attention mechanism (O(L) complexity): + +``` +S_t = S_{t-1} * α_t * (I - β_t * k_t * k_t^T) + β_t * v_t * k_t^T +``` + +All GatedDeltaNet operations are local compute -- the QKVZ/BA input +projections and output projection are replicated across TP ranks (not +TP-sharded) because their parameter count is small relative to the MLP/MoE +that follows, and the recurrent state makes TP complex. The mock models +this correctly: `Qwen3_5GatedDeltaNet.forward()` returns `Workload()` (empty). + +Raw `MockedParam` objects track full-size parameters for accurate DP gradient +sync sizing in the `step()` method. This achieves both communication accuracy +(zero TP collectives from DeltaNet layers) and parameter-count accuracy +(full-size weights counted for DP all-reduce sizing). + +Verified end-to-end (2025-06-15): + +``` +Qwen3.5-9B (32L, h=4096, TP=8): 82 fwd ops (24 DeltaNet x 2 MLP + 8 FullAttn x 4 + 2) +Qwen3-8B (36L, h=4096, TP=8): 146 fwd ops (36 layers x 4 + 2) +Observed reduction: 44% (64 fewer ops: 48 from DeltaNet + 16 from 4 fewer layers) +``` + +Per-layer comparison: + +| Layer type | Count | Attn comms | MLP comms | Per-layer | Total | +|---|---|---|---|---|---| +| Qwen3-8B (all full-attn) | 36 | 2 | 2 | 4 | 144 | +| Qwen3.5 FullAttention | 8 | 2 | 2 | 4 | 32 | +| Qwen3.5 GatedDeltaNet | 24 | **0** | 2 | 2 | 48 | +| Embedding + LM head | 2 | -- | -- | 1 | 2 | +| **Qwen3.5-9B total** | | | | | **82** | + +--- + +## Qwen3.5 Architecture Details + +### Hybrid Layer Layout + +``` +full_attention_interval = 4 → pattern: [L, L, L, F, L, L, L, F, ...] + +layer_types (from config.json): + ["linear_attention", "linear_attention", "linear_attention", "full_attention", ...] + +Layer selection: (layer_id + 1) % full_attention_interval == 0 → Full Attention +``` + +### Full Attention Layer Features + +- head_dim = 256 (2× Qwen3's 128) +- QK-Norm: per-head RMSNorm on Q and K before RoPE (same as Qwen3) +- partial_rotary_factor = 0.25 (only 64 of 256 dims receive RoPE) +- MRoPE: multimodal RoPE with mrope_section = [11, 11, 10] +- attn_output_gate = True: sigmoid gate on attention output +- Q projection doubled: half for query, half for gating signal + +All these are local compute — zero communication impact. + +### GatedDeltaNet Features + +- linear_key_head_dim = 128, linear_value_head_dim = 128 +- linear_num_key_heads = 16 (CONSTANT across all model sizes) +- linear_num_value_heads varies per model (16-64) +- linear_conv_kernel_dim = 4 (causal depthwise conv + SiLU) +- Uses RMSNormGated (not standard RMSNorm, not QK-Norm) + +### MoE Differences from Qwen3 + +| Feature | Qwen3 MoE | Qwen3.5 MoE | +|---|---|---| +| Experts | 128 | 256-512 | +| Top-K | 8 | 8 or 10 | +| Shared experts | None (removed) | 1 (always active) | +| Per-expert FFN dim | intermediate_size | moe_intermediate_size | +| Shared expert FFN | N/A | shared_expert_intermediate_size | + +--- + +## Verified Model Configs + +### Qwen3 Dense + +| Model | hidden | layers | Q heads | KV heads | intermediate | head_dim | tie_emb | +|---|---|---|---|---|---|---|---| +| 0.6B | 1024 | 28 | 16 | 8 | 3072 | 128 | true | +| 1.7B | 2048 | 28 | 16 | 8 | 6144 | 128 | true | +| 4B | 2560 | 36 | 32 | 8 | 9728 | 128 | true | +| 8B | 4096 | 36 | 32 | 8 | 12288 | 128 | false | +| 14B | 5120 | 40 | 40 | 8 | 17408 | 128 | false | +| 32B | 5120 | 64 | 64 | 8 | 25600 | 128 | false | + +Common: vocab_size=151936, rope_theta=1M, max_position=40960, qk_norm=hardcoded, +use_sliding_window=false, attention_bias=false, rms_norm_eps=1e-6. + +### Qwen3 MoE + +| Model | hidden | layers | Q heads | KV heads | intermediate | experts | topk | shared | +|---|---|---|---|---|---|---|---|---| +| 30B-A3B | 2048 | 48 | 32 | 4 | 6144 | 128 | 8 | none | +| 235B-A22B | 4096 | 94 | 64 | 4 | 12288 | 128 | 8 | none | + +### Qwen3.5 Dense + +| Model | hidden | layers | Q heads | KV heads | intermediate | head_dim | full:lin | tie_emb | +|---|---|---|---|---|---|---|---|---| +| 0.8B | 1024 | 24 | 8 | 2 | 3584 | 256 | 6:18 | true | +| 2B | 2048 | 24 | 8 | 2 | 6144 | 256 | 6:18 | true | +| 4B | 2560 | 32 | 16 | 4 | 9216 | 256 | 8:24 | true | +| 9B | 4096 | 32 | 16 | 4 | 12288 | 256 | 8:24 | false | +| 27B | 5120 | 64 | 24 | 4 | 17408 | 256 | 16:48 | false | + +Common: vocab_size=248320, rope_theta=10M, max_position=262144, +full_attention_interval=4, linear_key_head_dim=128, linear_value_head_dim=128, +linear_num_key_heads=16, linear_conv_kernel_dim=4, partial_rotary=0.25, +attn_output_gate=true, MRoPE mrope_section=[11,11,10]. + +### Qwen3.5 MoE + +| Model | hidden | layers | Q heads | KV heads | experts | topk | moe_ffn | shared_ffn | full:lin | +|---|---|---|---|---|---|---|---|---|---| +| 35B-A3B | 2048 | 40 | 16 | 2 | 256 | 8 | 512 | 512 | 10:30 | +| 122B-A10B | 3072 | 48 | 32 | 2 | 256 | 8 | 1024 | 1024 | 12:36 | +| 397B-A17B | 4096 | 60 | 32 | 2 | 512 | 10 | 1024 | 1024 | 15:45 | + +--- + +## Usage Examples + +```bash +# Qwen3-8B dense, TP=8 +python -m workload_generator.generate_megatron_workload \ + --frame Qwen3 --model_name Qwen3-8B \ + --hidden_size 4096 --num_hidden_layers 36 \ + --num_attention_heads 32 --num_key_value_heads 8 --head_dim 128 \ + --ffn_hidden_size 12288 --vocab_size 151936 \ + --world_size 8 --tensor_model_parallel_size 8 \ + --seq_length 4096 --micro_batch 2 \ + --enable_sequence_parallel --swiglu + +# Qwen3.5-9B dense, TP=8 +python -m workload_generator.generate_megatron_workload \ + --frame Qwen3.5 --model_name Qwen3.5-9B \ + --hidden_size 4096 --num_hidden_layers 32 \ + --num_attention_heads 16 --num_key_value_heads 4 --head_dim 256 \ + --ffn_hidden_size 12288 --vocab_size 248320 \ + --world_size 8 --tensor_model_parallel_size 8 \ + --seq_length 4096 --micro_batch 2 \ + --enable_sequence_parallel --swiglu + +# Qwen3-235B-A22B MoE, TP=8, EP=8 +python -m workload_generator.generate_megatron_workload \ + --frame Qwen3 --model_name Qwen3-235B-A22B \ + --hidden_size 4096 --num_hidden_layers 94 \ + --num_attention_heads 64 --num_key_value_heads 4 --head_dim 128 \ + --ffn_hidden_size 12288 --vocab_size 151936 \ + --world_size 64 --tensor_model_parallel_size 8 \ + --expert_model_parallel_size 8 \ + --num_experts 128 --moe_router_topk 8 \ + --moe_enable --enable_sequence_parallel --swiglu + +# Qwen3.5-397B-A17B MoE via config file +python -m workload_generator.generate_megatron_workload \ + --frame Qwen3.5 \ + --config path/to/qwen3_5_397b_config.json \ + --world_size 128 --tensor_model_parallel_size 8 \ + --expert_model_parallel_size 16 \ + --moe_enable --enable_sequence_parallel --swiglu +``` + +### Config JSON Format + +```json +{ + "hidden_size": 4096, + "intermediate_size": 12288, + "num_hidden_layers": 36, + "num_attention_heads": 32, + "num_key_value_heads": 8, + "head_dim": 128, + "vocab_size": 151936, + "moe_enable": false, + "model_name": "Qwen3-8B" +} +``` + +For Qwen3.5, add: `"full_attention_interval": 4`, `"linear_key_head_dim": 128`, +`"linear_value_head_dim": 128`, `"linear_num_key_heads": 16`, +`"linear_num_value_heads": 32`, `"linear_conv_kernel_dim": 4`, +`"moe_intermediate_size": 1024`, `"shared_expert_intermediate_size": 1024`. + +For HF Qwen3.5 multimodal format, `text_config` is unpacked automatically by +`Qwen3_5Params.__init__`. + +--- + +## Testing + +```bash +# Qwen3 mock smoke test (standalone) +cd aicb +.venv/bin/python workload_generator/mocked_model/training/MockedQwen3.py /path/to/config.json + +# Qwen3.5 mock smoke test (standalone) +.venv/bin/python workload_generator/mocked_model/training/MockedQwen3_5.py /path/to/config.json + +# Full workload generation (requires pandas, torch optional) +.venv/bin/python -m workload_generator.generate_megatron_workload \ + --frame Qwen3 --config config.json --workload_only +``` + +Config file keys use HuggingFace naming: `num_hidden_layers`, `intermediate_size`, +`vocab_size`. Training-specific keys (`tensor_model_parallel_size`, `seq_length`, +`micro_batch`) are layered on top by `Qwen3Params` / `Qwen3_5Params`. + +--- + +## Known Limitations + +1. **MTP (Multi-Token Prediction) not modeled.** Qwen3.5 has MTP heads after the + backbone. These add ~2% extra compute but negligible communication. Documented + as known gap. + +2. **MoE backward communication was undercounted** (fixed 2025-06-15). + MOEMLP.backward() in MockedMegatron.py was missing two `workloads.extend()` + calls on the return values of `self.permutation()` and `self.unpermutation()`. + Pre-existing since original commit. Fix restored backward-forward parity. + +3. **Inference workload mocks for Qwen3/Qwen3.5 are skeletal.** Only the + AIOB compute benchmarks exist for inference. Training workload mocks are + complete. + +4. **AIOB training benchmarks for Qwen3/Qwen3.5 do not exist.** Only inference + AIOB benchmarks are implemented (`inference/AiobQwen3Moe.py`, + `inference/AiobQwen3Next.py`). Training AIOB would require implementing + backward-pass kernels not present in the inference benchmarks. + +--- + +## Adding a New Model Family + +1. Create `training/Mocked.py` with: + - `Attention`: attention module (reuse ColumnLinear/RowLinear) + - `Mlp`: SwiGLU MLP (reuse ColumnLinear/RowLinear) + - `TransformerLayer`: attention + MLP composition + - `Model`: embedding + N layers + lm_head assembly + - `Params(MockedParamsBase)`: config loading with defaults + +2. Add Qwen3-specific CLI args to `utils/utils.py` via a `get__params` + function called from `get_params()`. + +3. Add `""` to `--frame` choices in `utils/utils.py`. + +4. Add import and dispatch in `generate_megatron_workload.py`: + ```python + from workload_generator.mocked_model.training.Mocked import Model + ... + elif args.frame == "": + model = Model(args) + ``` + +5. Test with config file and verify workload counts. The definitive formulas + (verified 2025-06-15 across all configs, TP=8, SP enabled): + + ``` + DENSE (TP only): + Megatron: fwd = 1 + L * 4 bwd = fwd * 1.50 + (MegatronModel.forward omits final_norm ColumnLinear; no +1 at end) + Megatron-8B: 1 + 32*4 = 129 + + Qwen3: fwd = 1 + L * 4 + 1 bwd = fwd * 1.50 + (Qwen3Model.forward includes lm_head ColumnLinear; +1 at end) + (4 per layer: attn QKV col + attn out row + MLP gate_up col + MLP down row) + Qwen3-8B: 1 + 36*4 + 1 = 146 + + Qwen3.5: fwd = 1 + L_f * 4 + L_d * 2 + 1 bwd = fwd * 1.50 + L_f = full-attention layers (L // full_attention_interval) + L_d = DeltaNet layers (L - L_f) + Qwen3.5-9B: 1 + 8*4 + 24*2 + 1 = 82 + + MOE (TP + EP, no shared experts): + Megatron: fwd = 1 + L * 7 bwd ≈ fwd + Qwen3: fwd = 1 + L * 7 + 1 bwd ≈ fwd + (7 per layer: 2 attn + 5 MoE dispatch/combine; MoE=preprocess+dispatch+perm+unperm+combine) + Qwen3-235B: 1 + 94*7 + 1 = 660 + + MOE (TP + EP, with shared experts — Qwen3.5): + fwd = 1 + L_f * 9 + L_d * 7 + 1 bwd ≈ fwd * 1.04 + FullAttn layers: 9 (2 attn + 2 shared expert + 5 MoE) + DeltaNet layers: 7 (0 attn + 2 shared expert + 5 MoE) + Qwen3.5-397B: 1 + 15*9 + 45*7 + 1 = 452 + ``` + + MoE per-layer breakdown (5 ops): preprocess all-gather + dispatch all-to-all + + TP all-gather + TP reduce-scatter + combine all-to-all. + Shared expert adds 2 ops: ColumnLinear all-gather + RowLinear reduce-scatter. + All MoE msg_size formulas divide by ep_size (fixed 2025-06-15). diff --git a/aicb/workload_generator/RESEARCH.md b/aicb/workload_generator/RESEARCH.md new file mode 100644 index 000000000..477f1294a --- /dev/null +++ b/aicb/workload_generator/RESEARCH.md @@ -0,0 +1,529 @@ +# Qwen3 and Qwen3.5 AICB Workload Generator -- Complete Research Report + +**Purpose**: Definitive reference for generating SimAI training workloads for +Qwen3 and Qwen3.5 model families. Covers all 22 supported models across 3 +families (LLaMA 3.1, Qwen3, Qwen3.5), both dense and MoE variants. + +**Scope**: Architecture comparison, verified parameter tables from HF config.json, +AICB integration strategy, workload communication formulas, training parallelism +configurations, bug fixes applied, generated workload CSV manifest, copy-paste +usage examples. + +**Confidence**: HIGH. All 19 model configs verified from config.json ground truth +(hf-mirror.com, modelscope.cn). All workload formulas verified exact-match +against running code (68 regression tests passing). Architecture confirmed +against Qwen3 Technical Report (arxiv 2505.09388) and Qwen3.5 official blog +(qwen-ai.com). HF Transformers source code (modeling_qwen3.py, modeling_qwen3_5.py) +verified via ghproxy.net. + +**For colleagues**: Start at Section 1 (Architecture Comparison) for the big +picture, Section 5 (Workload Formulas) for the math, and the "How to Use" +section at the end for copy-paste code. The CLAUDE.md in this directory is +the operational companion (read by AI agents automatically). + +Verified 2025-06-15. Primary external sources: hf-mirror.com (16 configs), +modelscope.cn (1 gated config), ar5iv.labs.arxiv.org/html/2505.09388 (paper), +qwen-ai.com/qwen-3-5 (blog), ghproxy.net (4 Transformers source files). + +--- + +## 1. Architecture Comparison + +| Feature | LLaMA 3 | Qwen3 | Qwen3.5 | +|---------|---------|-------|---------| +| Attention | GQA (derived head_dim) | GQA (explicit head_dim=128) | Hybrid: 75% DeltaNet + 25% GQA (head_dim=256) | +| QK-Norm | No | Yes (per-head RMSNorm before RoPE) | Full-attn: Yes. DeltaNet: RMSNormGated | +| RoPE theta | 500,000 | 1,000,000 | 10,000,000 | +| KV heads | varies (8-8) | fixed 8 (dense), 4 (MoE) | fixed 2-4 | +| Vocab | 128,000 | 151,936 | 248,320 | +| MoE shared expert | N/A | None | 1 per layer (sigmoid-gated) | +| MTP layers | No | No | 1 | +| Q_dim vs hidden | Always equal | 3 of 6 diverge | 3 of 5 diverge | + +Critical finding: Qwen3/3.5 use explicit head_dim (128 or 256). +num_q * head_dim frequently differs from hidden_size. This breaks the LLaMA +assumption that QKV projection = 3 * hidden_size. + +| Model | hidden | Q heads | Q dim | Mismatch | +|-------|--------|---------|-------|----------| +| Qwen3-0.6B | 1024 | 16 | 2048 | 2.0x | +| Qwen3-4B | 2560 | 32 | 4096 | 1.6x | +| Qwen3-32B | 5120 | 64 | 8192 | 1.6x | +| Qwen3-30B-A3B | 2048 | 32 | 4096 | 2.0x | +| Qwen3-235B-A22B | 4096 | 64 | 8192 | 2.0x | +| Qwen3.5-0.8B | 1024 | 8 | 2048 | 2.0x | +| Qwen3.5-27B | 5120 | 24 | 6144 | 1.2x | +| Qwen3.5-397B-A17B | 4096 | 32 | 8192 | 2.0x | + +8 of 16 models diverge. All MoE models diverge by 2x. + +--- + +## 2. Verified Model Configurations + +All 16 configs verified from primary-source config.json files. + +### 2.1 Qwen3 Dense (6 variants, MockedQwen3.py, moe_enable=False) + +| Model | h | L | Q | KV | FFN | vocab | max_seq | tie | +|-------|---|---|----|-----|-----|-------|---------|-----| +| 0.6B | 1024 | 28 | 16 | 8 | 3072 | 151936 | 40960 | yes | +| 1.7B | 2048 | 28 | 16 | 8 | 6144 | 151936 | 40960 | yes | +| 4B | 2560 | 36 | 32 | 8 | 9728 | 151936 | 40960 | yes | +| 8B | 4096 | 36 | 32 | 8 | 12288 | 151936 | 40960 | no | +| 14B | 5120 | 40 | 40 | 8 | 17408 | 151936 | 40960 | no | +| 32B | 5120 | 64 | 64 | 8 | 25600 | 151936 | 40960 | no | + +Common: head_dim=128, rope_theta=1M, rms_norm_eps=1e-6, qk_norm=yes, +use_sliding_window=false, attention_bias=false. Qwen3-14B has 40 layers +and Qwen3-32B has 64 heads (OLMo-core PR listing 48L, 40H was incorrect). + +### 2.2 Qwen3 MoE (2 variants, MockedQwen3.py, moe_enable=True) + +| Model | h | L | Q | KV | FFN | moe_ffn | exp | topk | shared | vocab | +|-------|---|---|----|-----|-----|---------|-----|------|--------|-------| +| 30B-A3B | 2048 | 48 | 32 | 4 | 6144 | 768 | 128 | 8 | none | 151936 | +| 235B-A22B | 4096 | 94 | 64 | 4 | 12288 | 1536 | 128 | 8 | none | 151936 | + +Common: head_dim=128, rope_theta=1M, decoder_sparse_step=1. + +Resolved: intermediate_size (6144/12288) is inherited but UNUSED. Verified from +HF source: Qwen3MoeDecoderLayer creates dense MLP then immediately replaces it +with SparseMoeBlock using moe_intermediate_size. Dense MLP is phantom-allocated +then garbage collected. Every layer is pure MoE: attention + sparse FFN only. + +### 2.3 Qwen3.5 Dense (5 variants, MockedQwen3_5.py, moe_enable=False) + +| Model | h | L | Q | KV | FFN | vocab | full:lin | tie | +|-------|---|---|----|-----|-----|-------|----------|-----| +| 0.8B | 1024 | 24 | 8 | 2 | 3584 | 248320 | 6:18 | yes | +| 2B | 2048 | 24 | 8 | 2 | 6144 | 248320 | 6:18 | yes | +| 4B | 2560 | 32 | 16 | 4 | 9216 | 248320 | 8:24 | yes | +| 9B | 4096 | 32 | 16 | 4 | 12288 | 248320 | 8:24 | no | +| 27B | 5120 | 64 | 24 | 4 | 17408 | 248320 | 16:48 | no | + +Common: head_dim=256, rope_theta=10M, max_pos=262144, full_attn_interval=4, +linear_key_head_dim=128, linear_value_head_dim=128, linear_num_key_heads=16 +(constant), linear_conv_kernel_dim=4, partial_rotary=0.25, attn_output_gate=true, +mrope_interleaved=true, mtp_num_hidden_layers=1. Linear V heads scale: 16 (0.8B, +2B), 32 (4B, 9B), 48 (27B). + +### 2.4 Qwen3.5 MoE (3 variants, MockedQwen3_5.py, moe_enable=True) + +| Model | h | L | Q | KV | hdim | exp | topk | moe_ffn | shared_ffn | lin_V | full:lin | vocab | +|-------|---|---|----|-----|------|-----|------|---------|------------|-------|----------|-------| +| 35B-A3B | 2048 | 40 | 16 | 2 | 256 | 256 | 8 | 512 | 512 | 32 | 10:30 | 248320 | +| 122B-A10B | 3072 | 48 | 32 | 2 | 256 | 256 | 8 | 1024 | 1024 | 64 | 12:36 | 248320 | +| 397B-A17B | 4096 | 60 | 32 | 2 | 256 | 512 | 10 | 1024 | 1024 | 64 | 15:45 | 248320 | + +Common: rope_theta=10M, full_attn_interval=4, linear_num_key_heads=16, +linear_conv_kernel_dim=4, partial_rotary=0.25, attn_output_gate=true, +mrope_interleaved=true, mtp_num_hidden_layers=1, model_type=qwen3_5_moe. +Shared expert: 1 per layer, sigmoid-gated, operates on ALL tokens (not routed). +FP8 native (blog: ~50% activation memory reduction). 262K native context, 1M via YaRN. + +Note: The A17B suffix in 397B-A17B refers to ~17B activated parameters per token +(10 routed experts + 1 shared expert, each with moe_ffn=1024), not 17 experts. +The Qwen3.5 blog incorrectly states "17 active" -- config.json confirms topk=10. + +--- + +## 3. Primary Source Verification + +| # | Model | Source | +|---|-------|--------| +| 1 | Qwen3-0.6B | gitcode.com config.json | +| 2 | Qwen3-1.7B | hf-mirror.com config.json (Base) | +| 3 | Qwen3-4B | hf-mirror.com config.json | +| 4 | Qwen3-8B | gitcode.com config.json | +| 5 | Qwen3-14B | hf-mirror.com config.json | +| 6 | Qwen3-32B | hf-mirror.com config.json | +| 7 | Qwen3-30B-A3B | hf-mirror.com config.json | +| 8 | Qwen3-235B-A22B | hf-mirror.com config.json | +| 9 | Qwen3.5-0.8B | gitcode.com config.json | +| 10 | Qwen3.5-2B | hf-mirror.com config.json | +| 11 | Qwen3.5-4B | gitcode.com config.json | +| 12 | Qwen3.5-9B | gitcode.com config.json | +| 13 | Qwen3.5-27B | gitcode.com config.json | +| 14 | Qwen3.5-35B-A3B | hf-mirror.com config.json | +| 15 | Qwen3.5-122B-A10B | hf-mirror.com config.json | +| 16 | Qwen3.5-397B-A17B | hf-mirror.com config.json | + +Code sources: modeling_qwen3_moe.py v4.51.0, modeling_qwen3_5_moe.py main. + +--- + +## 4. AICB Implementation + +### 4.1 File Status + +| File | Status | Variants | +|------|--------|----------| +| training/MockedQwen3.py | COMPLETE | Dense (6) + MoE (2) | +| training/MockedQwen3_5.py | COMPLETE | Dense (5) + MoE (3) | +| generate_megatron_workload.py | COMPLETE | --frame Qwen3 / Qwen3.5 | +| training/MockedMegatron.py | FIXED | MoE ep_size bug | +| training/MockedDeepSeek.py | FIXED | MoE ep_size bug | + +### 4.2 Key Design Decisions + +GatedDeltaNet: forward() returns Workload() (empty). All projections replicated +across TP. Parameters tracked via raw MockedParam for DP gradient sync. + +Qwen3Attention: uses explicit head_dim and GQA-aware num_kv_heads. ColumnLinear/ +RowLinear abstractions are dimension-agnostic, so TP communication sizes are +correct for all Q-dimension configurations. + +MoE: Qwen3 uses MOEMLP (128 experts, top-8, no shared). Qwen3.5 uses +Qwen3_5MoEMLP (256-512 experts, top-8/10, 1 shared expert with learned gate). + +--- + +## 5. Communication Formulas + +### 5.1 Forward pass (TP=8, SP enabled) + +``` +Qwen3 dense: fwd = 1 + L * 4 + 1 (emb + layers*4 + lm_head) +Qwen3 MoE: fwd = 1 + L * 7 + 1 (emb + layers*7 + lm_head) +Qwen3.5 dense: fwd = 1 + Lf*4 + Ld*2 + 1 +Qwen3.5 MoE: fwd = 1 + Lf*9 + Ld*7 + 1 +``` + +Lf = L / 4, Ld = L - Lf. + +### 5.2 Per-layer composition + +| Layer type | Attn | MLP/MoE | Total | Components | +|---|---|---|---|---| +| Qwen3 standard | 2 | 2 | 4 | QKV(ag)+out(rs)+gate_up(ag)+down(rs) | +| Qwen3 MoE | 2 | 5 | 7 | +preprocess(ag)+dispatch(a2a)+perm(ag)+unperm(rs)+combine(a2a) | +| Qwen3.5 FullAttn | 2 | 2 or 2+5 | 4 or 9 | Same as Qwen3 | +| Qwen3.5 DeltaNet | 0 | 2 or 2+5 | 2 or 7 | MLP/MoE only | + +ag=all_gather, rs=reduce_scatter, a2a=all_to_all + +MoE msg_size formulas (divide by ep_size, fixed 2025-06-15): +``` +dispatch: seq_len * hidden * batch * topk / tp / ep * 2 +permute: 2 * hidden * topk * batch * seq_len / ep +combine: same as dispatch +unpermute: same as permute +``` + +### 5.3 Verified across all 16 variants + +| Model | L | Type | Fwd | +|-------|---|------|-----| +| Qwen3-0.6B | 28 | Dense | 114 | +| Qwen3-1.7B | 28 | Dense | 114 | +| Qwen3-4B | 36 | Dense | 146 | +| Qwen3-8B | 36 | Dense | 146 | +| Qwen3-14B | 40 | Dense | 162 | +| Qwen3-32B | 64 | Dense | 258 | +| Qwen3-30B-A3B | 48 | MoE | 338 | +| Qwen3-235B-A22B | 94 | MoE | 660 | +| Qwen3.5-0.8B | 24 | Dense | 62 | +| Qwen3.5-2B | 24 | Dense | 62 | +| Qwen3.5-4B | 32 | Dense | 82 | +| Qwen3.5-9B | 32 | Dense | 82 | +| Qwen3.5-27B | 64 | Dense | 162 | +| Qwen3.5-35B-A3B | 40 | MoE | 302 | +| Qwen3.5-122B-A10B | 48 | MoE | 362 | +| Qwen3.5-397B-A17B | 60 | MoE | 452 | + +DeepSeekV3 regression passed (538 fwd). Megatron/LLaMA 3.1 verified: 8B=129, 70B=321, +405B MoE=883. + +Generated workload CSVs in `results/mocked_workload/` (18 files, 9 models): +- LLaMA: llama8b, llama_405b_moe +- Qwen3: qwen3_8b, qwen3_30b_a3b, qwen3_235b_a22b +- Qwen3.5: qwen35_9b, qwen35_35b_a3b, qwen35_122b_a10b, qwen35_397b_a17b + +--- + +## 6. Training Parallelism Configurations + +### 6.1 Verified (from official sources) + +| Model | Source | TP | PP | EP | CP | Total GPUs | +|-------|--------|----|----|----|----|------------| +| Qwen3-235B-A22B (pretrain) | NVIDIA Megatron-Bridge recipe | **4** | **16** | **8** | **2** | 4x16x8 = 512 | +| Qwen3-235B-A22B (SFT) | NVIDIA Megatron-Bridge recipe | 4 | -- | 4 | -- | 64 (8 nodes) | + +### 6.2 Estimated (from NVIDIA node counts, not verified) + +| Model | Source | Nodes | GPUs | Likely TP/EP | Rationale | +|-------|--------|-------|------|-------------|-----------| +| Qwen3.5-35B-A3B | NVIDIA Megatron-Bridge | 2 | 16 | TP=2, EP=8 | 35B total, 3B active, 256 experts | +| Qwen3.5-122B-A10B | NVIDIA Megatron-Bridge | 4 | 32 | TP=4, EP=8 | 122B total, 10B active, 256 experts | +| Qwen3.5-397B-A17B | Not published | -- | -- | TP=4, EP=16? | 397B total, 17B active, 512 experts, topk=10 | + +### 6.3 Data Gap + +Qwen3.5 was released Feb-Mar 2026. As of 2025-06, no technical report or +pretraining paper exists for Qwen3.5. The NVIDIA Megatron-Bridge docs provide +inference node counts but not explicit TP/PP/EP training breakdowns. Estimates +above assume EP scaling with expert count (Qwen3: 128e/EP8; Qwen3.5: 256e/EP8 +or 512e/EP16) and TP scaling with active parameters (~4-8 per active-param +tier). These should be treated as placeholders until official documentation +is published. + +### 5.4 Head-to-head communication comparison + +| Comparison | Model A | Items | Model B | Items | Reduction | +|---|---|---|---|---|---| +| Dense equiv | Qwen3-8B | 146 | Qwen3.5-9B | 82 | 44% | +| MoE equiv | Qwen3-30B | 338 | Qwen3.5-35B | 302 | 11% | +| Flagship | Qwen3-235B | 660 | Qwen3.5-397B | 452 | 32% | + +The 44% dense reduction comes from 24 DeltaNet layers generating zero attention +communication. The smaller MoE reduction reflects MoE all-to-all dominating +per-layer communication, with Qwen3.5 adding shared expert TP overhead. + +--- + +## 6. Bugs Found and Fixed + +| # | Bug | Files | Impact | Fix | +|---|-----|-------|--------|-----| +| 1 | MoE msg_size overstated by ep_size (12 formulas) | MOEMLP, DeepSeekMoE, Qwen3_5MoEMLP | 64x overcount at EP=64; 50 GB phantom comm/iter at 30B | Added // ep_size | +| 2 | Qwen3 MoE not supported | MockedQwen3.py | 30B-A3B, 235B unusable | Added moe_enable + MOEMLP | +| 3 | DeltaNet overcounted TP comm (4/layer vs 0 correct) | MockedQwen3_5.py | 59% overestimate in 9B | Reverted to comm-accurate | +| 4 | OLMo-core PR wrong configs (14B: 48L, 32B: 40H) | Research | Wrong third-party specs | Primary config.json: 40L, 64H | +| 5 | CLAUDE.md described fixed bugs as current | CLAUDE.md | Stale docs | Updated all sections | + +--- + +## 7. Known Limitations + +1. MTP not modeled (~2% extra compute, negligible communication). +2. Inference mocks for Qwen3/Qwen3.5 are skeletal (no forward/backward). +3. AIOB training benchmarks for Qwen3/Qwen3.5 now EXIST (added 2025-06-15): + - AiobQwen3.py: GQA + QK-Norm attention, SwiGLU MLP. No special GPU deps. + - AiobQwen3_5.py: GatedDeltaNet (75%) + gated full-attention (25%). Requires + flash-linear-attention + causal-conv1d on CUDA. Graceful FLA_AVAILABLE=False + fallback on CPU/MPS (structure only, no timings). + - Not yet run on GPU hardware -- benchmark structure complete, timings pending. +4. GatedDeltaNet parameter count approximate (linear weights only). +5. Qwen3Params training orchestration fields (pp_rank, use_distributed_optimizer) + set by CLI get_params(), not by Params defaults. + +--- + +## Qwen3.5 Official Blog (qwen-ai.com/qwen-3-5, March 2026) + +The Qwen3.5 launch blog confirms several architectural findings and adds +performance context: + +- **Hybrid ratio**: 75% GatedDeltaNet + 25% gated attention (3:1), matching + config.json `full_attention_interval=4` and `layer_types` pattern. +- **Performance**: 8.6x faster decode at 32K tokens, 19x at 256K vs Qwen3-Max. + Community: 35B-A3B hits 196 tok/s on RTX 4090. +- **Vocab**: 248,320 tokens (up from 152K in Qwen3), 201 languages. +- **FP8 native**: ~50% activation memory reduction. +- **All models multimodal**: Text, image, video via early fusion. No separate + "VL" suffix -- every model handles all modalities. +- **Context**: 262K native, 1M via YaRN. +- **MoE 397B**: 512 total experts, 10 active + 1 shared (confirmed by config.json + `num_experts_per_tok=10`). Blog says "17 active" -- this conflates the activated + parameter count (~17B, hence the A17B suffix) with the number of routed experts. +- **Thinking control**: API parameter `enable_thinking` (not chat template + toggles like Qwen3). On by default for 27B+, off for smaller. +- **Deployment caveats**: Ollama broken as of March 2026 (infinite CoT loops, + broken tool calling, 5-7x slower than llama.cpp). QLoRA 4-bit incompatible + with hybrid architecture (use bf16 LoRA instead). + +Source: https://qwen-ai.com/qwen-3-5 (fetched 2025-06-15, 38KB). + +--- + +## Qwen3 Technical Report (arxiv 2505.09388, May 2025) + +Fetched via ar5iv.labs.arxiv.org HTML (140KB). The paper covers all 8 Qwen3 +models and confirms the architectural findings from config.json. + +### Architecture confirmation + +- Paper Table 1 (dense): Layers, Q/KV Heads, Tie Embedding, Context Length for + all 6 dense models. Matches config.json exactly. +- Paper Table 2 (MoE): Layers, Q/KV Heads, Experts (128/8), Context Length. + Matches config.json. +- QK-Norm replaces QKV-bias from Qwen2: "remove QKV-bias used in Qwen2 and + introduce QK-Norm to the attention mechanism." Confirms QK-Norm is architectural, + not configurable. +- MoE: "128 total experts with 8 activated per token. Unlike Qwen2.5-MoE, the + Qwen3-MoE design excludes shared experts." +- Tokenizer: BBPE, vocabulary 151,669 (config.json has 151,936 -- the difference + is special tokens). + +### Training strategy + +Three-stage process: +1. **General Stage (S1)**: 30T+ tokens at seq_len=4096. 119 languages. Full + language proficiency and world knowledge. +2. **Reasoning Stage (S2)**: ~5T tokens at seq_len=4096. Higher proportion of + STEM/code/reasoning/synthetic data. Faster LR decay. +3. **Long Context Stage**: Hundreds of billions of tokens at seq_len=32768. + 75% of text between 16384-32768 tokens, 25% between 4096-16384. + +Total: 36 trillion tokens. RoPE theta increased from 10,000 to 1,000,000 via +ABF technique. YARN + Dual Chunk Attention for 4x inference context extension. + +### What the paper does NOT include + +- Exact global batch size, micro_batch, learning rate schedule values +- Optimizer hyperparameters (beta1, beta2, epsilon, weight decay) +- GPU count, parallelism configuration (TP/PP/DP) +- These must be sourced from NVIDIA Megatron-Bridge recipes (TP=4/PP=16/EP=8/CP=2 + for 235B) and Megatron conventions (~4M tokens global batch). + +Source: https://ar5iv.labs.arxiv.org/html/2505.09388 (fetched 2025-06-15, 140KB). + +--- + +## How to Use + +### Quick smoke test (standalone Python, no GPU needed) + +```bash +cd aicb +python3 workload_generator/mocked_model/training/MockedQwen3.py +python3 workload_generator/mocked_model/training/MockedQwen3_5.py +``` + +### Generate workload CSVs programmatically + +```python +from workload_generator.mocked_model.training.MockedQwen3 import Qwen3Params, Qwen3Model + +# Qwen3-8B dense +cfg = Qwen3Params() +cfg.hidden_size = 4096 +cfg.intermediate_size = 12288 +cfg.num_hidden_layers = 36 +cfg.num_attention_heads = 32 +cfg.num_key_value_heads = 8 +cfg.head_dim = 128 +cfg.vocab_size = 151936 +cfg.tensor_model_parallel_size = 8 +cfg.world_size = 8 +cfg.seq_length = 4096 +cfg.micro_batch = 2 +cfg.enable_sequence_parallel = True + +model = Qwen3Model(cfg) +fwd = model.forward() # 146 ops +bwd = model.backward() # 219 ops +fwd.dump("qwen3_8b_fwd") # -> results/mocked_workload/qwen3_8b_fwd_workload.csv +``` + +### Qwen3.5-9B dense + +```python +from workload_generator.mocked_model.training.MockedQwen3_5 import Qwen3_5Params, Qwen3_5Model + +cfg = Qwen3_5Params() +cfg.hidden_size = 4096; cfg.intermediate_size = 12288 +cfg.num_hidden_layers = 32; cfg.num_attention_heads = 16 +cfg.num_key_value_heads = 4; cfg.head_dim = 256 +cfg.vocab_size = 248320 +cfg.full_attention_interval = 4 +cfg.linear_key_head_dim = 128; cfg.linear_value_head_dim = 128 +cfg.linear_num_key_heads = 16; cfg.linear_num_value_heads = 32 +cfg.linear_conv_kernel_dim = 4 +cfg.tensor_model_parallel_size = 8; cfg.world_size = 8 +cfg.seq_length = 4096; cfg.micro_batch = 2 +cfg.enable_sequence_parallel = True + +model = Qwen3_5Model(cfg) +fwd = model.forward() # 82 ops +bwd = model.backward() # 123 ops +``` + +### Qwen3.5-397B-A17B MoE + +```python +cfg = Qwen3_5Params() +cfg.hidden_size = 4096; cfg.intermediate_size = 12288 +cfg.moe_intermediate_size = 1024; cfg.shared_expert_intermediate_size = 1024 +cfg.num_hidden_layers = 60; cfg.num_attention_heads = 32 +cfg.num_key_value_heads = 2; cfg.head_dim = 256 +cfg.vocab_size = 248320 +cfg.full_attention_interval = 4 +cfg.linear_key_head_dim = 128; cfg.linear_value_head_dim = 128 +cfg.linear_num_key_heads = 16; cfg.linear_num_value_heads = 64 +cfg.linear_conv_kernel_dim = 4 +cfg.tensor_model_parallel_size = 4; cfg.expert_model_parallel_size = 16 +cfg.world_size = 128 +cfg.seq_length = 4096; cfg.micro_batch = 2 +cfg.enable_sequence_parallel = True +cfg.moe_enable = True; cfg.num_experts = 512; cfg.moe_router_topk = 10 + +model = Qwen3_5Model(cfg) +fwd = model.forward() # 452 ops +bwd = model.backward() # 468 ops +``` + +### CLI (requires --frame, --swiglu, --head_dim, --num_key_value_heads) + +```bash +python -m workload_generator.generate_megatron_workload \ + --frame Qwen3 --model_name Qwen3-8B \ + --hidden_size 4096 --num_layers 36 \ + --num_attention_heads 32 --num_key_value_heads 8 --head_dim 128 \ + --intermediate_size 12288 --vocab_size 151936 \ + --world_size 8 --tensor_model_parallel_size 8 \ + --seq_length 4096 --micro_batch 2 \ + --enable_sequence_parallel --swiglu --workload_only +``` + +--- + +## Source Audit (complete, 2025-06-15) + +### External HTTP (24 succeeded, 4 blocked) + +**Config.json (17 models):** +- hf-mirror.com: Qwen3-0.6B, 1.7B, 4B, 8B, 14B, 32B, 30B-A3B +- hf-mirror.com: Qwen3.5-0.8B, 2B, 4B, 9B, 27B, 35B-A3B, 122B-A10B, 397B-A17B +- hf-mirror.com: Qwen3.5-2B (initial fetch) +- modelscope.cn: Qwen3-235B-A22B (gated on HF, available via ModelScope API) + +**Architecture papers / docs:** +- ar5iv.labs.arxiv.org/html/2505.09388 -- Qwen3 Technical Report (140KB HTML) +- arxiv.org/abs/2505.09388 -- Qwen3 paper abstract (55KB HTML) +- qwen-ai.com/qwen-3-5 -- Qwen3.5 official blog (38KB, March 2026) +- docs.nvidia.com/nemo/megatron-bridge/0.4.0/models/vlm/qwen35-vl.html -- NVIDIA Megatron-Bridge Qwen3.5 config + +**HF Transformers source code (4 files via ghproxy.net):** +- modeling_qwen3.py (528 lines) +- modeling_qwen3_5.py (~2100 lines) +- modeling_llama.py (~530 lines) +- modular_qwen3.py (~80 lines) + +**Blocked (Fastly CDN 151.101.x.x timeout):** +- huggingface.co, qwenlm.github.io, github.com, raw.githubusercontent.com, + arxiv.org PDF, sciencestack.ai, deepwiki.com, aihub.caict.ac.cn + +### Local files read (18) + +MockedMegatron.py, MockedQwen3.py, MockedQwen3_5.py, MockedDeepSeek.py, +AiobMegatron.py, SimAI_training_workload_generator.py, generate_megatron_workload.py, +workload_generator.py, utils/utils.py, MockedModel.py, CLAUDE.md, +gpu_compute_timing.txt, workload_analytical.txt, workload_gpt13b_128g.txt, +busbw.yaml, test_mocked_qwen3.py, test_mocked_qwen3_5.py, log_analyzer/log.py + +### Files created (4) + +RESEARCH.md (450 lines), test_regression_bugs.py (325 lines), +AiobQwen3.py (338 lines), AiobQwen3_5.py (409 lines). + +### Test suite + +138 tests passing (run from `aicb/` directory): + - test_regression_bugs.py: 10 tests (MOEMLP backward, DeepSeekMoE backward, + GatedDeltaNet empty, SwiGLU compat, CLI aliases) + - test_mocked_qwen3.py: 58 tests (dense + MoE comm counts, QK-Norm, tie embedding) + - test_mocked_qwen3_5.py: 55 tests (dense comm counts, SimAI generator format) + - test_logitem_ranks.py, test_rank_mapper.py: 15 tests (pre-existing) diff --git a/aicb/workload_generator/SimAI_inference_workload_generator.py b/aicb/workload_generator/SimAI_inference_workload_generator.py new file mode 100644 index 000000000..8bb342e5e --- /dev/null +++ b/aicb/workload_generator/SimAI_inference_workload_generator.py @@ -0,0 +1,362 @@ +import sys +import os +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import workload_generator.mocked_model.MockedModel +import workload_generator.mocked_model.inference.MockedDeepSeek as MockedDeepSeek +import workload_generator.mocked_model.inference.MockedQwen3Moe as MockedQwen3Moe +import workload_generator.mocked_model.inference.MockedQwen3Next as MockedQwen3Next +from workload_generator.mocked_model.MockedModel import InferencePhase +from utils.utils import CommType, get_params, get_comp_out, extract_inference_averages, Strategy, get_ep_expected_m_per_group +import os +from typing import List, Tuple +from collections import deque +import dataclasses +from enum import Enum + +try: + import torch +except ImportError as e: + torch = None + print("Failed to import 'torch'.") +import math +import re + +@dataclasses.dataclass +class Work_Item: + name: str = dataclasses.field(default="none") + placeholder: int = dataclasses.field(default=-1) + forward_compute_time: int = dataclasses.field(default=0) + forward_comm: str = dataclasses.field(default="NONE") + forward_comm_size: str = dataclasses.field(default="0") + backward_compute_time: int = dataclasses.field(default=0) + backward_comm: str = dataclasses.field(default="NONE") + backward_comm_size: int = dataclasses.field(default=0) + dp_compute_time: int = dataclasses.field(default=0) + dp_comm: str = dataclasses.field(default="NONE") + dp_comm_size: int = dataclasses.field(default=0) + process_time: int = dataclasses.field(default=100) + + + +def _get_aiob_compute_time(compute_cache, forward_or_backward, stage, dowarn=True, aiob_enable=True): + default_compute_time = 1 + if not aiob_enable: + return default_compute_time + compute_time_map = compute_cache + if stage == "shared_experts" or stage == "dense_mlp": + prefix = "mlp" + else: + prefix = stage + + for key, value in compute_time_map.items(): + if prefix == key: + + compute_time = compute_time_map.get(key) + return compute_time + + if dowarn: + print(f"[warn] can't match any stage with {stage}, using default_compute_time {default_compute_time}") + return default_compute_time + +class LayerInfo: + def __init__(self, layer_id, layer_name): + self.layer_id = layer_id + self.layer_name = layer_name +def _get_model_details(model): + layers = [] + visited = set() + + def traverse_model(model, child_id=0): + if id(model) in visited: + return + visited.add(id(model)) + if ( + isinstance(model, MockedDeepSeek.DeepSeekAttention) + or isinstance(model, MockedDeepSeek.DeepSeekMLP) + # or isinstance(model, MockedDeepSeek.DeepSeekMOE) + # or isinstance(model, MockedDeepSeek.DeepSeekColumnLinear) + # or isinstance(model, MockedDeepSeek.DeepSeekRowLinear) + or isinstance(model, MockedQwen3Moe.Qwen3MoeRMSNorm) + or isinstance(model, MockedQwen3Moe.Qwen3MoeAttention) + or isinstance(model, MockedQwen3Moe.Qwen3MoeRoute) + or isinstance(model, MockedQwen3Moe.Qwen3MoeExpert) + or isinstance(model, MockedQwen3Next.Qwen3NextRMSNorm) + or isinstance(model, MockedQwen3Next.Qwen3NextAttention) + or isinstance(model, MockedQwen3Next.Qwen3NextGatedDeltaNet) + or isinstance(model, MockedQwen3Next.Qwen3NextRoute) + or isinstance(model, MockedQwen3Next.Qwen3NextExpert) + ): + layers.append(LayerInfo(model.layer_id, model.name)) + for child in model.child_modules(): + traverse_model(child, child_id+1) + + traverse_model(model) + + return layers + +class SimAIWorkload(): + def __init__(self, model, args, compue_cache=None): + self.model = model + self.args = args + self.compute_cache = compute_cache + self.workload = [] + self.seq_len = args.seq_length + self.tp = args.tensor_model_parallel_size + self.mbs = args.micro_batch + self.batch_size = args.micro_batch # 添加缺失的 batch_size 属性 + if args.moe_enable: + self.expert_model_parallel_size = args.expert_model_parallel_size + self.num_experts = args.num_experts + #如果有moe_router_topk则用，否则用num_experts_per_tok + self.topk = args.moe_router_topk if hasattr(args,"moe_router_topk") else args.num_experts_per_tok + + + def get_comm_size(self): + phase = getattr(self.args, "phase", InferencePhase.DECODE.value) + if phase == InferencePhase.DECODE.value: + m = self.batch_size + elif phase == InferencePhase.PREFILL.value: + m = self.args.seq_length + tp_comm_size = 2 * m * self.args.hidden_size + ep_combine_size = tp_comm_size * self.topk // self.tp + ep_dispatch_size = ep_combine_size + + if any(s in self.args.frame for s in ('DeepSeek', 'Qwen3')): + # for DeepEP based on https://github.com/parthpower/DeepEP/commit/50aee15f592bc22142eb04b7d718296b19613ae9 + ep_dispatch_size = int(ep_dispatch_size * MockedDeepSeek.FP8_FACTOR) + + return tp_comm_size, ep_dispatch_size, ep_combine_size + + + def workload_generate_aiob(self): + default_compute_time = 1 + compute_time = 0 + tp_comm_size, ep_dispatch_size, ep_combine_size = self.get_comm_size() + layers = _get_model_details(self.model) + print("layer length:" , len(layers)) + + for layer in layers: + name = layer.layer_name + if args.tensor_model_parallel_size == 1 : + forward_comm = "NONE" + else: + forward_comm = "ALLREDUCE" + + if "norm" in name or "shared_expert" in name: # comp only layers + compute_time = _get_aiob_compute_time( + self.compute_cache, "forward", name + ) + self.workload.append( + Work_Item( + name=name, + forward_compute_time=compute_time, + forward_comm="NONE", + forward_comm_size="0", + backward_compute_time=0, + backward_comm="NONE", + backward_comm_size=0, + dp_compute_time=0, + dp_comm="NONE", + dp_comm_size=0, + ) + ) + elif "attention" in name or "dense_mlp" in name: # comp & ALLREDUCE comm layers + compute_time = _get_aiob_compute_time( + self.compute_cache, "forward", name + ) + self.workload.append( + Work_Item( + name=name, + forward_compute_time=compute_time, + forward_comm=forward_comm, + forward_comm_size=tp_comm_size, + backward_compute_time=0, + backward_comm="NONE", + backward_comm_size=0, + dp_compute_time=0, + dp_comm="NONE", + dp_comm_size=0, + ) + ) + elif "moe_route" in name: # moe route + compute_time = _get_aiob_compute_time( + self.compute_cache, "forward", name + ) + self.workload.append( + Work_Item( + name="moe_route", + forward_compute_time=compute_time, + forward_comm="ALLTOALL_EP", + # forward_comm_size = "2*" + str(token_num) + "*" + str(self.args.moe_router_topk) + "*" + str(self.args.expert_dim), + forward_comm_size = ep_dispatch_size, + backward_compute_time=default_compute_time, + backward_comm="NONE", + backward_comm_size=0, + dp_compute_time=default_compute_time, + dp_comm="NONE", + dp_comm_size=0 + ) + ) + + elif "moe_expert" in name: # moe experrt + compute_time = _get_aiob_compute_time( + self.compute_cache, "forward", name + ) + + if self.args.frame == "DeepSeek": + #TODO currently AiobDeepSeek doesn't support moe_route, should be fixed. + self.workload.append( + Work_Item( + name="moe_route", + forward_compute_time=default_compute_time, + forward_comm="ALLTOALL_EP", + # forward_comm_size = "2*" + str(token_num) + "*" + str(self.args.moe_router_topk) + "*" + str(self.args.expert_dim), + forward_comm_size = ep_dispatch_size, + backward_compute_time=default_compute_time, + backward_comm="NONE", + backward_comm_size=0, + dp_compute_time=default_compute_time, + dp_comm="NONE", + dp_comm_size=0 + ) + ) + self.workload.append( + Work_Item( + name="moe_expert", + forward_compute_time=compute_time, + forward_comm="ALLTOALL_EP", + forward_comm_size = ep_combine_size, + backward_compute_time=default_compute_time, + backward_comm="NONE", + backward_comm_size=0, + dp_compute_time=default_compute_time, + dp_comm="NONE", + dp_comm_size=0 + ) + ) + def dump_file(self, filename): + filename = filename + ".txt" + + pp_comm_value = 2 * self.mbs * self.seq_len * self.args.hidden_size * (1 if self.args.pipeline_model_parallel > 1 else 0) + + pp_comm = ( + f"pp_comm: {pp_comm_value}" + ) + with open(filename, "w") as f: + f.write(( + f"HYBRID_TRANSFORMER_FWD_IN_BCKWD model_parallel_NPU_group: {self.args.tensor_model_parallel_size} " + f"ep: {self.args.expert_model_parallel_size} " + f"pp: {self.args.pipeline_model_parallel} " + f"all_gpus: {self.args.world_size} " + f"mode: 1 " # inference + f"vpp: 1 ga: 1 checkpoints: 0 checkpoint_initiates: 0 " + ) + pp_comm + "\n") + + f.write(str(len(self.workload)) + "\n") + for item in self.workload: + f.write( + "\t".join([str(getattr(item, k)) for k in item.__dict__.keys()]) + + "\n" + ) + + + +if __name__ == "__main__": + import sys + import argparse + from utils.utils import get_params + + # Parse command line arguments + parser = argparse.ArgumentParser(description="Generate inference workload for AI models") + parser.add_argument("model_name", help="Model name (e.g., DeepSeek-671B, Qwen3-Moe-235B, Qwen3-Next-80B)") + parser.add_argument("config_file", nargs="?", help="Path to JSON config file") + + # Add arguments that has default value + parser.add_argument("--aiob_enable", action="store_true", default=False, help="Enable AIOB") + parser.add_argument("--aiob_forward_loops", type=int, default=1, help="Number of AIOB forward loops") + parser.add_argument("--moe_routing_strategy", default=Strategy.RoundRobin.value, choices=[s.value for s in Strategy], help="MoE routing strategy") + parser.add_argument("--seq_length", type=int, default=1, help="Sequence length") + parser.add_argument("--micro_batch", type=int, default=1, help="Micro batch size") + parser.add_argument("--world_size", type=int, default=1, help="World size") + parser.add_argument("--tensor_model_parallel_size", default=1, type=int, help="Tensor model parallel size") + parser.add_argument("--expert_model_parallel_size", default=1, type=int, help="Expert model parallel size") + parser.add_argument("--pipeline_model_parallel", default=1, type=int, help="Pipeline model parallel size") + parser.add_argument("--moe_enable", default=True, action="store_true", help="Enable MoE") + parser.add_argument("--result_dir", default="results/workload/", help="Result directory") + parser.add_argument("--phase", + choices=[InferencePhase.DECODE.value, InferencePhase.PREFILL.value], + default=InferencePhase.DECODE.value, help="Inference phase") + + args = parser.parse_args() + + model_name = args.model_name + config_file = args.config_file + + if "Qwen3-Moe" in model_name: + args = MockedQwen3Moe.Qwen3MoeParams(config_file, args) + model = MockedQwen3Moe.Qwen3MoeModel(args) + elif "Qwen3-Next" in model_name: + args = MockedQwen3Next.Qwen3NextParams(config_file, args) + model = MockedQwen3Next.Qwen3NextModel(args) + elif "DeepSeek" in model_name: + args = MockedDeepSeek.DeepSeekParams(config_file, args) + model = MockedDeepSeek.DeepSeekModel(args) + else: + print(f"Invalid model name: {model_name}") + sys.exit(1) + # args = MockedDeepSeek.DeepSeekParams(config_file) + # model = MockedDeepSeek.DeepSeekModel(args) + phase = getattr(args, "phase", InferencePhase.DECODE.value) + result_dir = args.result_dir + if not os.path.isdir(result_dir): + os.makedirs(result_dir) + filename = f"{args.model_name}-world_size{args.world_size}-tp{args.tensor_model_parallel_size}-pp{args.pipeline_model_parallel}-ep{args.expert_model_parallel_size}-bs{args.micro_batch}-seq{args.seq_length}-{phase}" + + if args.aiob_enable: + if "Qwen3-Moe" in model_name: + import workload_generator.mocked_model.inference.AiobQwen3Moe as AiobQwen3Moe + aiob_model = AiobQwen3Moe.Qwen3MoeModel(args) + aiob_output_filepath = aiob_model() + elif "Qwen3-Next" in model_name: + import workload_generator.mocked_model.inference.AiobQwen3Next as AiobQwen3Next + aiob_model = AiobQwen3Next.Qwen3NextModel(args) + aiob_output_filepath = aiob_model() + elif "DeepSeek" in model_name: + import workload_generator.mocked_model.inference.AiobDeepSeek as AiobDeepSeek + aiob_model = AiobDeepSeek.DeepSeekModel(args) + aiob_output_filepath = aiob_model() + else: + print(f"Invalid model name: {model_name}") + sys.exit(1) + + else: + aiob_dir = "results/aiob_outputs" + aiob_output_filename = f"{args.model_name}-world_size{args.world_size}-tp{args.tensor_model_parallel_size}-pp{args.pipeline_model_parallel}-ep{args.expert_model_parallel_size}-bpg{args.micro_batch}-seq{args.seq_length}-{phase}.txt" + aiob_output_filepath = os.path.join(aiob_dir,aiob_output_filename) + if not os.path.exists(aiob_output_filepath): + print(f"aiob not enabled, and {aiob_output_filepath} not found. Using default compute time.") + aiob_output_filepath = "" + else: + print(f"aiob not enabled, using existing file {aiob_output_filepath}.") + compute_cache = extract_inference_averages(aiob_output_filepath,args) + print("compute_cache = {") + for key, value in compute_cache.items(): + print(f" '{key}' : {value},") + print("}") + + work = SimAIWorkload( + model, args,compute_cache + ) + name_layers = work.workload_generate_aiob() + # set comm_size = 0 for any comm_type == NONE + for i in range(len(work.workload)): + if work.workload[i].forward_comm == "NONE": + work.workload[i].forward_comm_size = 0 + if work.workload[i].backward_comm == "NONE": + work.workload[i].backward_comm_size = 0 + + filepath = os.path.join(result_dir, filename) + work.dump_file(filepath) + print("workload save in :", filepath) + print("Finish Model initialization") \ No newline at end of file diff --git a/aicb/workload_generator/SimAI_training_workload_generator.py b/aicb/workload_generator/SimAI_training_workload_generator.py new file mode 100755 index 000000000..83058ce65 --- /dev/null +++ b/aicb/workload_generator/SimAI_training_workload_generator.py @@ -0,0 +1,1013 @@ +""" +Copyright (c) 2021, Alibaba Group; +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from workload_generator.mocked_model.training import MockedDeepSeek +import workload_generator.mocked_model.training.MockedDeepspeed +from workload_generator.mocked_model.training.MockedMegatron import * +from workload_generator.mocked_model.training.MockedDeepSeek import * +from workload_generator.mocked_model.training.MockedQwen3 import ( + Qwen3Model, Qwen3Attention, Qwen3Mlp, Qwen3Embedding, Qwen3RMSNorm, +) +from workload_generator.mocked_model.training.MockedQwen3_5 import Qwen3_5Model +from workload_generator.mocked_model.training.MockedQwen3_5 import ( + Qwen3_5RowLinear, Qwen3_5ColumnLinear, Qwen3_5RMSNorm, + Qwen3_5Embedding, Qwen3_5FullAttention, Qwen3_5Mlp, +) +from workload_generator.mocked_model.MockedModel import MockedParam, MockedModel +from utils.utils import CommType, get_params, get_comp_out, extract_averages +import os +from typing import List, Tuple +from collections import deque +import dataclasses +from enum import Enum + +try: + import torch +except ImportError as e: + torch = None + print("Failed to import 'torch'.") +import math +import re + + + + + +@dataclasses.dataclass +class Work_Item: + name: str = dataclasses.field(default="none") + placeholder: int = dataclasses.field(default=-1) + forward_compute_time: int = dataclasses.field(default=0) + forward_comm: str = dataclasses.field(default="NONE") + forward_comm_size: int = dataclasses.field(default=0) + backward_compute_time: int = dataclasses.field(default=0) + backward_comm: str = dataclasses.field(default="NONE") + backward_comm_size: int = dataclasses.field(default=0) + dp_compute_time: int = dataclasses.field(default=0) + dp_comm: str = dataclasses.field(default="NONE") + dp_comm_size: int = dataclasses.field(default=0) + process_time: int = dataclasses.field(default=100) + + + +def _get_aiob_compute_time(compute_cache, forward_or_backward, stage, dowarn=True): + compute_time_map = compute_cache + + # if compute time with exact layer name exist, use it without prefix + if stage in compute_time_map.keys(): + return compute_time_map[stage] + + if stage == "grad": + prefix = stage + "_" + forward_or_backward + elif stage == "embedding": + prefix = "Emb" + elif stage == "final": + prefix = "attention" + "_" + forward_or_backward + else: + prefix = stage + "_" + forward_or_backward + + for key, value in compute_time_map.items(): + if prefix == key: + + compute_time = compute_time_map.get(key) + return compute_time + # just so it doesn't spam warning when trying to get per-layer comp time + if dowarn: + print("[warn] can't match any stage", stage) + return 1 + + +class LayerInfo: + def __init__(self, layer_id, layer_name, param_count): + self.layer_id = layer_id + self.layer_name = layer_name + self.param_count = param_count + + +class SIMAI_workload: + def __init__(self, model, args, compute_cache=None): + self.model = model + self.args = args + self.compute_cache = compute_cache + self.workload = [] + self.seq_len = args.seq_length + self.tp = args.tensor_model_parallel_size + self.mbs = args.micro_batch + if args.moe_enable: + self.expert_model_parallel_size = args.expert_model_parallel_size + self.num_experts = args.num_experts + self.topk = args.moe_router_topk + + def get_model_details(self): + layers = [] + visited = set() + + def traverse_model(model): + if id(model) in visited: + return + visited.add(id(model)) + + if self.args.enable_sequence_parallel: + if ( + isinstance(model, MegatronColumnLinear) + or isinstance(model, MegatronRowLinear) + or isinstance(model, MegatronEmbedding) + or isinstance(model, FusedLayernorm) + or isinstance(model, DeepSeekLinear) + or isinstance(model, Qwen3Embedding) + or isinstance(model, Qwen3RMSNorm) + or isinstance(model, Qwen3_5RowLinear) + or isinstance(model, Qwen3_5ColumnLinear) + or isinstance(model, Qwen3_5Embedding) + or isinstance(model, Qwen3_5RMSNorm) + ): + params = model.parameters() + param_count = sum(p.numel() for p in params) + layers.append(LayerInfo(model.layer_id, model.name, param_count)) + if isinstance(model, MOEMLP) or isinstance(model, DeepSeekMoE): + moe_params = model.parameters() + moe_param_count = sum(p.numel() for p in moe_params) + layers.append(LayerInfo(model.layer_id, model.name, moe_param_count)) + + else: + if ( + isinstance(model, MegatronAttention) + or isinstance(model, MegatronMlp) + or isinstance(model, MegatronEmbedding) + or isinstance(model, DeepSeekMLA) + or isinstance(model, DeepSeekMoE) + or isinstance(model, Qwen3Attention) + or isinstance(model, Qwen3Mlp) + or isinstance(model, Qwen3Embedding) + or isinstance(model, Qwen3_5FullAttention) + or isinstance(model, Qwen3_5Mlp) + or isinstance(model, Qwen3_5Embedding) + ): + params = model.parameters() + param_count = sum(p.numel() for p in params) + layers.append(LayerInfo(model.layer_id, model.name, param_count)) + + for child in model.child_modules(): + traverse_model(child) + + traverse_model(self.model) + + return layers + + def _get_total_params(self): + total_params = 0 + moe_param_count = 0 + layers = self.get_model_details() + for layer in layers: + total_params += layer.param_count + if "moe" in layer.layer_name: + moe_param_count += layer.param_count + + return total_params, moe_param_count + + def workload_generate_aiob(self): + # args.world_size --> total gpus number + self.ga_num = self.args.global_batch // (self.args.micro_batch * self.args.dp_num) + if self.ga_num < 1: + print( + "[WARN]: ga num < 1, please confirm global_batch num and micro_batch num" + ) + default_compute_time = 1 + compute_time = 0 + tp_comm_size = ( + 2 * self.args.micro_batch * self.args.seq_length * self.args.hidden_size + ) + layers = self.get_model_details() + total_params, moe_param_count = self._get_total_params() + # self.workload.append(Work_Item(name="norm", forward_compute_time=0, + # forward_comm = "BROADCAST", forward_comm_size= 8*self.args.micro_batch*self.args.seq_length, + # backward_compute_time=default_compute_time, backward_comm="NONE", backward_comm_size=0, + # dp_compute_time=default_compute_time, dp_comm="NONE", dp_comm_size=0 + # )) + forward_compute_time = _get_aiob_compute_time( + self.compute_cache, "forward", "grad" + ) + backward_compute_time = _get_aiob_compute_time( + self.compute_cache, "backward", "grad" + ) + self.workload.append( + Work_Item( + name="grad_gather", + forward_compute_time=default_compute_time, + forward_comm="NONE", + forward_comm_size=0, + backward_compute_time=default_compute_time, + backward_comm="NONE", + backward_comm_size=0, + dp_compute_time=default_compute_time, + dp_comm="ALLGATHER", + dp_comm_size=2 * (total_params-moe_param_count), + ) + ) + self.workload.append( + Work_Item( + name="grad_param_comm", + forward_compute_time=default_compute_time, + forward_comm="NONE", + forward_comm_size=0, + backward_compute_time=default_compute_time, + backward_comm="NONE", + backward_comm_size=0, + dp_compute_time=default_compute_time, + dp_comm="REDUCESCATTER", + dp_comm_size=4 * (total_params-moe_param_count), + ) + ) + self.workload.append( + Work_Item( + name="grad_param_compute", + forward_compute_time=default_compute_time, + forward_comm="NONE", + forward_comm_size=0, + backward_compute_time=forward_compute_time + backward_compute_time, + backward_comm="NONE", + backward_comm_size=0, + dp_compute_time=default_compute_time, + dp_comm="NONE", + dp_comm_size=0, + ) + ) + + if not self.args.enable_sequence_parallel: + self.workload.append( + Work_Item( + name="layernorm", + forward_compute_time=default_compute_time, + forward_comm="NONE", + forward_comm_size=0, + backward_compute_time=default_compute_time, + backward_comm="ALLREDUCE", + backward_comm_size=2 * total_params, + dp_compute_time=default_compute_time, + dp_comm="NONE", + dp_comm_size=0, + ) + ) + if args.tensor_model_parallel_size == 1 : + emd_backward_comm = "NONE" + else: + emd_backward_comm = "ALLREDUCE" + self.workload.append( + Work_Item( + name="embedding_grads", + forward_compute_time=default_compute_time, + forward_comm="NONE", + forward_comm_size=0, + backward_compute_time=default_compute_time, + backward_comm=emd_backward_comm, + backward_comm_size=tp_comm_size, + dp_compute_time=default_compute_time, + dp_comm="NONE", + dp_comm_size=0, + ) + ) + if self.args.expert_model_parallel_size != self.args.dp_num: + self.workload.append(Work_Item(name="moe_grad_norm1", forward_compute_time=default_compute_time, + forward_comm = "NONE", forward_comm_size= 0, + backward_compute_time=default_compute_time, backward_comm="NONE", backward_comm_size=0, + dp_compute_time=default_compute_time, dp_comm="ALLGATHER_DP_EP", dp_comm_size=2*moe_param_count + )) + self.workload.append(Work_Item(name="moe_grad_norm2", forward_compute_time=default_compute_time, + forward_comm = "NONE", forward_comm_size= 0, + backward_compute_time=default_compute_time, backward_comm="NONE", backward_comm_size=0, + dp_compute_time=default_compute_time, dp_comm="REDUCESCATTER_DP_EP", dp_comm_size=4*moe_param_count + )) + for _ in range(self.ga_num): + for layer in layers: + name = layer.layer_name + forward_comm = backward_comm = backward_comm_2 = "NONE" + forward_comm_size = tp_comm_size + emb_comm_size = tp_comm_size + backward_comm_size = 0 + dp_comm = "NONE" + dp_comm_size = 0 + + # try get layer specific compute time + # e.g. from AiobDeepSeek.DeepSeekMLA's compute times + layer_comp_time = _get_aiob_compute_time( + self.compute_cache, "", name, False + ) + # _get_aiob_compute_time return 1 in case no compute time found + if layer_comp_time == 1: + layer_comp_time = None + + if self.args.enable_sequence_parallel: + if "embedding" in name: + if args.tensor_model_parallel_size == 1 : + forward_comm = "NONE" + backward_comm = "NONE" + else: + forward_comm = "ALLREDUCE" + backward_comm = "NONE" + emb_compute_time = _get_aiob_compute_time( + self.compute_cache, "", "embedding" + ) + self.workload.append( + Work_Item( + name=name, + forward_compute_time=emb_compute_time, + forward_comm=forward_comm, + forward_comm_size=emb_comm_size , + backward_compute_time=default_compute_time, + backward_comm=backward_comm, + backward_comm_size=backward_comm_size, + dp_compute_time=backward_compute_time, + dp_comm=dp_comm, + dp_comm_size=dp_comm_size, + ) + ) + if "attention_linear" in name: + # for non-shareded linear in attention block + + # similar to row linear but without comms + if layer_comp_time != None: + forward_compute_time = layer_comp_time + backward_compute_time = layer_comp_time + else: + forward_compute_time=_get_aiob_compute_time( + self.compute_cache, "forward", name.split("_")[0] + ) + backward_compute_time = _get_aiob_compute_time( + self.compute_cache, "backward", name.split("_")[0] + ) + if self.args.recompute_activations: + forward_compute_time *= 2 + self.workload.append( + Work_Item( + name=name, + forward_compute_time=forward_compute_time, + forward_comm="NONE", + forward_comm_size=0, + backward_compute_time=backward_compute_time, + backward_comm="NONE", + backward_comm_size=0,#sp overlap allgather + dp_compute_time=backward_compute_time, + dp_comm=dp_comm, + dp_comm_size=dp_comm_size, + ) + ) + if "row" in name: + + if layer_comp_time != None: + forward_compute_time = layer_comp_time + backward_compute_time = layer_comp_time + else: + forward_compute_time = _get_aiob_compute_time( + self.compute_cache, "forward", name.split("_")[0] + ) + backward_compute_time = _get_aiob_compute_time( + self.compute_cache, "backward", name.split("_")[0] + ) + + if self.args.recompute_activations and 'attention' in name: + forward_compute_time *= 2 + forward_compute_time = int(forward_compute_time / 2) + backward_compute_time = int(backward_compute_time / 2) + forward_comm_size_sp = tp_comm_size + if args.tensor_model_parallel_size == 1 : + forward_comm = "NONE" + backward_comm = "NONE" + else: + forward_comm = "REDUCESCATTER" + backward_comm = "ALLGATHER" + self.workload.append( + Work_Item( + name=name, + forward_compute_time=forward_compute_time, + forward_comm=forward_comm, + forward_comm_size=forward_comm_size, + backward_compute_time=backward_compute_time, + backward_comm=backward_comm, + backward_comm_size=forward_comm_size_sp,#sp overlap allgather + dp_compute_time=backward_compute_time, + dp_comm=dp_comm, + dp_comm_size=dp_comm_size, + ) + ) + + elif "column" in name: + if layer_comp_time != None: + forward_compute_time = layer_comp_time + backward_compute_time = layer_comp_time + else: + forward_compute_time = _get_aiob_compute_time( + self.compute_cache, "forward", name.split("_")[0] + ) + backward_compute_time = _get_aiob_compute_time( + self.compute_cache, "backward", name.split("_")[0] + ) + + if self.args.recompute_activations and 'attention' in name: + forward_compute_time *= 2 + forward_compute_time = int(forward_compute_time / 2) + backward_compute_time = int(backward_compute_time / 2) + if args.tensor_model_parallel_size == 1 : + forward_comm = "NONE" + backward_comm = "NONE" + backward_comm_2 = "NONE" + else: + forward_comm = "ALLGATHER" + backward_comm = "REDUCESCATTER" + backward_comm_2 = "ALLGATHER" + self.workload.append( + Work_Item( + name=name, + forward_compute_time=forward_compute_time, + forward_comm=forward_comm, + forward_comm_size=forward_comm_size, + backward_compute_time=backward_compute_time, + backward_comm=backward_comm, + backward_comm_size=backward_comm_size, + dp_compute_time=backward_compute_time, + dp_comm=dp_comm, + dp_comm_size=dp_comm_size, + ) + ) + elif "moelayer" in name: + if layer_comp_time != None: + forward_compute_time = layer_comp_time + backward_compute_time = layer_comp_time + else: + forward_compute_time = _get_aiob_compute_time( + self.compute_cache, "forward", name.split("_")[0] + ) + backward_compute_time = _get_aiob_compute_time( + self.compute_cache, "backward", name.split("_")[0] + ) + + # MoE Communication is based on Megatron core_v0.13.0:https://github.com/NVIDIA/Megatron-LM/blob/core_v0.13.0 + forward_comm1 = "ALLGATHER" # for EP + forward_comm2 = "ALLTOALL_EP" + forward_comm3 = "ALLGATHER" + forward_comm4 = "REDUCESCATTER" + forward_comm5 = "ALLTOALL_EP" + if args.expert_model_parallel_size == 1: + forward_comm2 = "NONE" + forward_comm5 = "NONE" + if args.tensor_model_parallel_size == 1: + if args.expert_model_parallel_size == 1: + forward_comm1 = "NONE" + forward_comm3 = "NONE" + forward_comm4 = "NONE" + + # if args.expert_model_parallel_size != 1: + ep_allgather_size = 2 * self.expert_model_parallel_size * self.num_experts * self.tp + fwd_ep_dispatch_size = tp_comm_size * self.topk // self.tp + bkwd_ep_dispatch_size = tp_comm_size * self.topk // self.tp + ep_combine_size = tp_comm_size * self.topk // self.tp + + if self.args.frame == "DeepSeek": + # for DeepEP based on https://github.com/parthpower/DeepEP/commit/50aee15f592bc22142eb04b7d718296b19613ae9 + # only fprop does the FP8 + fwd_ep_dispatch_size = int(fwd_ep_dispatch_size * MockedDeepSeek.FP8_FACTOR) + # rest of the comm shapes are similar to megatron + # EP All gather + self.workload.append(Work_Item(name=name, forward_compute_time=forward_compute_time, + forward_comm = forward_comm1, forward_comm_size=ep_allgather_size, + backward_compute_time=backward_compute_time, backward_comm="NONE", backward_comm_size=0, + dp_compute_time=default_compute_time, dp_comm=dp_comm, dp_comm_size=dp_comm_size + )) + # EP dispatch + self.workload.append(Work_Item(name=name, forward_compute_time=default_compute_time, + forward_comm = forward_comm2, forward_comm_size=fwd_ep_dispatch_size, + backward_compute_time=default_compute_time, backward_comm=forward_comm2, backward_comm_size=bkwd_ep_dispatch_size, + dp_compute_time=default_compute_time, dp_comm=dp_comm, dp_comm_size=dp_comm_size + )) + # TP All reduce + self.workload.append(Work_Item(name=name, forward_compute_time=default_compute_time, + forward_comm = forward_comm3, forward_comm_size=tp_comm_size*self.topk, + backward_compute_time=default_compute_time, backward_comm=forward_comm4, backward_comm_size=tp_comm_size*self.topk, + dp_compute_time=default_compute_time, dp_comm=dp_comm, dp_comm_size=dp_comm_size + )) + self.workload.append(Work_Item(name=name, forward_compute_time=default_compute_time, + forward_comm = forward_comm4, forward_comm_size=tp_comm_size*self.topk, + backward_compute_time=default_compute_time, backward_comm=forward_comm3, backward_comm_size=tp_comm_size*self.topk, + dp_compute_time=default_compute_time, dp_comm=dp_comm, dp_comm_size=dp_comm_size + )) + # EP combine + self.workload.append(Work_Item(name=name, forward_compute_time=default_compute_time, + forward_comm = forward_comm5, forward_comm_size=ep_combine_size, + backward_compute_time=default_compute_time, backward_comm=forward_comm5, backward_comm_size=ep_combine_size, + dp_compute_time=default_compute_time, dp_comm=dp_comm, dp_comm_size=dp_comm_size + )) + else: + if args.tensor_model_parallel_size == 1 : + forward_comm = "NONE" + backward_comm = "NONE" + else: + + forward_comm = "ALLREDUCE" + backward_comm = "NONE" + if self.args.recompute_activations and 'attention' in name: + forward_compute_time *= 2 + if "embedding" in name: + emb_compute_time = _get_aiob_compute_time( + self.compute_cache, "", "embedding" + ) + self.workload.append( + Work_Item( + name=name, + forward_compute_time=emb_compute_time, + forward_comm=forward_comm, + forward_comm_size=forward_comm_size, + backward_compute_time=default_compute_time, + backward_comm=backward_comm, + backward_comm_size=backward_comm_size, + dp_compute_time=backward_compute_time, + dp_comm=dp_comm, + dp_comm_size=dp_comm_size, + ) + ) + else: + if layer_comp_time != None: + forward_compute_time = layer_comp_time + backward_compute_time = layer_comp_time + else: + forward_compute_time = _get_aiob_compute_time( + self.compute_cache, "forward", name.split("_")[0] + ) + backward_compute_time = _get_aiob_compute_time( + self.compute_cache, "backward", name.split("_")[0] + ) + self.workload.append( + Work_Item( + name=name, + forward_compute_time=forward_compute_time, + forward_comm=forward_comm, + forward_comm_size=forward_comm_size, + backward_compute_time=backward_compute_time, + backward_comm=backward_comm, + backward_comm_size=backward_comm_size, + dp_compute_time=backward_compute_time, + dp_comm=dp_comm, + dp_comm_size=dp_comm_size, + ) + ) + # compute_time = _get_aiob_compute_time(self.compute_cache, "forward", "embedding") + # self.workload.append(Work_Item(name="embedding_norm", forward_compute_time=compute_time, + # forward_comm = "ALLREDUCE", forward_comm_size= self.args.vocab_size*self.args.hidden_size*2, + # backward_compute_time=default_compute_time, backward_comm="NONE", backward_comm_size=0, + # dp_compute_time=default_compute_time, dp_comm="NONE", dp_comm_size=0 + # )) + for i in range(3): + self.workload.append( + Work_Item( + name="cross_entropy" + str(i + 1), + forward_compute_time=compute_time, + forward_comm="ALLREDUCE", + forward_comm_size=self.args.seq_length * self.args.micro_batch * 4, + backward_compute_time=compute_time, + backward_comm="NONE", + backward_comm_size=0, + dp_compute_time=compute_time, + dp_comm="NONE", + dp_comm_size=0, + ) + ) + + for i in range(4): + self.workload.append( + Work_Item( + name="optimizer" + str(i + 1), + forward_compute_time=compute_time, + forward_comm="ALLREDUCE", + forward_comm_size=4, + backward_compute_time=compute_time, + backward_comm="NONE", + backward_comm_size=0, + dp_compute_time=compute_time, + dp_comm="NONE", + dp_comm_size=0, + ) + ) + + def workload_generate(self): + # args.world_size --> total gpus number + self.ga_num = self.args.global_batch // (self.args.micro_batch * self.args.dp_num) + if self.ga_num < 1: + print( + "[WARN]: ga num < 1, please confirm global_batch num and micro_batch num" + ) + default_compute_time = 1 + compute_time = 0 + tp_comm_size = ( + 2 * self.args.micro_batch * self.args.seq_length * self.args.hidden_size + ) + layers = self.get_model_details() + total_params, moe_param_count = self._get_total_params() + # print(f"Total params is {total_params}, moe params is {moe_param_count}") + # self.workload.append(Work_Item(name="norm", forward_compute_time=0, + # forward_comm = "BROADCAST", forward_comm_size= 8*self.args.micro_batch*self.args.seq_length, + # backward_compute_time=default_compute_time, backward_comm="NONE", backward_comm_size=0, + # dp_compute_time=default_compute_time, dp_comm="NONE", dp_comm_size=0 + # )) + forward_compute_time = default_compute_time + backward_compute_time = default_compute_time + self.workload.append( + Work_Item( + name="grad_norm", + forward_compute_time=forward_compute_time, + forward_comm="ALLGATHER", + forward_comm_size=2 * total_params, + backward_compute_time=backward_compute_time, + backward_comm="NONE", + backward_comm_size=0, + dp_compute_time=default_compute_time, + dp_comm="REDUCESCATTER", + dp_comm_size=4 * total_params, + ) + ) + if not self.args.enable_sequence_parallel: + self.workload.append( + Work_Item( + name="layernorm", + forward_compute_time=default_compute_time, + forward_comm="NONE", + forward_comm_size=0, + backward_compute_time=default_compute_time, + backward_comm="ALLREDUCE", + backward_comm_size=2 * total_params, + dp_compute_time=default_compute_time, + dp_comm="NONE", + dp_comm_size=0, + ) + ) + if self.args.expert_model_parallel_size != self.args.dp_num: + self.workload.append(Work_Item(name="moe_grad_norm1", forward_compute_time=default_compute_time, + forward_comm = "NONE", forward_comm_size= 0, + backward_compute_time=default_compute_time, backward_comm="NONE", backward_comm_size=0, + dp_compute_time=default_compute_time, dp_comm="ALLGATHER_DP_EP", dp_comm_size=2*moe_param_count + )) + self.workload.append(Work_Item(name="moe_grad_norm2", forward_compute_time=default_compute_time, + forward_comm = "NONE", forward_comm_size= 0, + backward_compute_time=default_compute_time, backward_comm="NONE", backward_comm_size=0, + dp_compute_time=default_compute_time, dp_comm="REDUCESCATTER_DP_EP", dp_comm_size=4*moe_param_count + )) + for _ in range(self.ga_num): + for layer in layers: + name = layer.layer_name + forward_comm = backward_comm = backward_comm_2 = "NONE" + forward_comm_size = tp_comm_size + backward_comm_size = tp_comm_size + dp_comm = "NONE" + dp_comm_size = 0 + if self.args.enable_sequence_parallel: + if "embedding" in name: + self.workload.append( + Work_Item( + name=name, + forward_compute_time=default_compute_time, + forward_comm=forward_comm, + forward_comm_size=forward_comm_size, + backward_compute_time=default_compute_time, + backward_comm=backward_comm, + backward_comm_size=backward_comm_size, + dp_compute_time=backward_compute_time, + dp_comm=dp_comm, + dp_comm_size=dp_comm_size, + ) + ) + if "attention_linear" in name: + # for non-shareded linear in attention block + + # similar to row linear but without comms + forward_compute_time=_get_aiob_compute_time( + self.compute_cache, "forward", name.split("_")[0] + ) + backward_compute_time = _get_aiob_compute_time( + self.compute_cache, "backward", name.split("_")[0] + ) + if self.args.recompute_activations: + forward_compute_time *= 2 + self.workload.append( + Work_Item( + name=name, + forward_compute_time=forward_compute_time, + forward_comm="NONE", + forward_comm_size=0, + backward_compute_time=backward_compute_time, + backward_comm="NONE", + backward_comm_size=0,#sp overlap allgather + dp_compute_time=backward_compute_time, + dp_comm=dp_comm, + dp_comm_size=dp_comm_size, + ) + ) + if "row" in name: + if self.args.recompute_activations and 'attention' in name: + forward_comm_size *= 2 + forward_comm = "REDUCESCATTER" + backward_comm = "ALLGATHER" + self.workload.append(Work_Item(name=name, forward_compute_time=default_compute_time, + forward_comm = forward_comm, forward_comm_size= forward_comm_size, + backward_compute_time=default_compute_time, backward_comm="NONE", backward_comm_size=tp_comm_size, + dp_compute_time=default_compute_time, dp_comm=dp_comm, dp_comm_size=dp_comm_size + )) + if "column" in name: + if self.args.recompute_activations and 'attention' in name: + forward_comm_size *= 2 + forward_comm = "ALLGATHER" + forward_comm2 = "NONE" + backward_comm = "REDUCESCATTER" + backward_comm_2 = "ALLGATHER" + self.workload.append(Work_Item(name=name, forward_compute_time=default_compute_time, + forward_comm = forward_comm, forward_comm_size= forward_comm_size, + backward_compute_time=default_compute_time, backward_comm="NONE", backward_comm_size=0, + dp_compute_time=default_compute_time, dp_comm=dp_comm, dp_comm_size=dp_comm_size + )) + if "moelayer" in name: + forward_comm1 = "ALLGATHER" # for EP + forward_comm2 = "ALLTOALL_EP" + forward_comm3 = "ALLGATHER" + forward_comm4 = "REDUCESCATTER" + forward_comm5 = "ALLTOALL_EP" + # if args.expert_model_parallel_size != 1: + ep_allgather_size = 2 * self.expert_model_parallel_size * self.num_experts * self.tp + fwd_ep_dispatch_size = tp_comm_size * self.topk // self.tp + bkwd_ep_dispatch_size = tp_comm_size * self.topk // self.tp + ep_combine_size = tp_comm_size * self.topk // self.tp + + if self.args.frame == "DeepSeek": + # for DeepEP based on https://github.com/parthpower/DeepEP/commit/50aee15f592bc22142eb04b7d718296b19613ae9 + # only fprop does the FP8 + fwd_ep_dispatch_size = int(fwd_ep_dispatch_size * MockedDeepSeek.FP8_FACTOR) + # rest of the comm shapes are similar to megatron + # EP All gather + self.workload.append(Work_Item(name=name, forward_compute_time=default_compute_time, + forward_comm = forward_comm1, forward_comm_size=ep_allgather_size, + backward_compute_time=default_compute_time, backward_comm="NONE", backward_comm_size=0, + dp_compute_time=default_compute_time, dp_comm=dp_comm, dp_comm_size=dp_comm_size + )) + # EP dispatch + self.workload.append(Work_Item(name=name, forward_compute_time=default_compute_time, + forward_comm = forward_comm2, forward_comm_size=fwd_ep_dispatch_size, + backward_compute_time=default_compute_time, backward_comm=forward_comm2, backward_comm_size=bkwd_ep_dispatch_size, + dp_compute_time=default_compute_time, dp_comm=dp_comm, dp_comm_size=dp_comm_size + )) + # TP All reduce + self.workload.append(Work_Item(name=name, forward_compute_time=default_compute_time, + forward_comm = forward_comm3, forward_comm_size=tp_comm_size*self.topk, + backward_compute_time=default_compute_time, backward_comm=forward_comm4, backward_comm_size=tp_comm_size*self.topk, + dp_compute_time=default_compute_time, dp_comm=dp_comm, dp_comm_size=dp_comm_size + )) + self.workload.append(Work_Item(name=name, forward_compute_time=default_compute_time, + forward_comm = forward_comm4, forward_comm_size=tp_comm_size*self.topk, + backward_compute_time=default_compute_time, backward_comm=forward_comm3, backward_comm_size=tp_comm_size*self.topk, + dp_compute_time=default_compute_time, dp_comm=dp_comm, dp_comm_size=dp_comm_size + )) + # EP combine + self.workload.append(Work_Item(name=name, forward_compute_time=default_compute_time, + forward_comm = forward_comm5, forward_comm_size=ep_combine_size, + backward_compute_time=default_compute_time, backward_comm=forward_comm5, backward_comm_size=ep_combine_size, + dp_compute_time=default_compute_time, dp_comm=dp_comm, dp_comm_size=dp_comm_size + )) + + else: + forward_comm = "ALLREDUCE" + backward_comm = "ALLREDUCE" + if self.args.recompute_activations and 'attention' in name: + forward_comm_size *= 2 + if "embedding" in name: + self.workload.append( + Work_Item( + name=name, + forward_compute_time=default_compute_time, + forward_comm=forward_comm, + forward_comm_size=forward_comm_size, + backward_compute_time=default_compute_time, + backward_comm=backward_comm, + backward_comm_size=backward_comm_size, + dp_compute_time=backward_compute_time, + dp_comm=dp_comm, + dp_comm_size=dp_comm_size, + ) + ) + else: + self.workload.append( + Work_Item( + name=name, + forward_compute_time=default_compute_time, + forward_comm=forward_comm, + forward_comm_size=forward_comm_size, + backward_compute_time=default_compute_time, + backward_comm=backward_comm, + backward_comm_size=backward_comm_size, + dp_compute_time=default_compute_time, + dp_comm=dp_comm, + dp_comm_size=dp_comm_size, + ) + ) + self.workload.append( + Work_Item( + name="embedding_norm", + forward_compute_time=default_compute_time, + forward_comm="ALLREDUCE", + forward_comm_size=self.args.vocab_size * self.args.hidden_size * 2, + backward_compute_time=default_compute_time, + backward_comm="NONE", + backward_comm_size=0, + dp_compute_time=default_compute_time, + dp_comm="NONE", + dp_comm_size=0, + ) + ) + for i in range(3): + self.workload.append( + Work_Item( + name="cross_entropy" + str(i + 1), + forward_compute_time=compute_time, + forward_comm="ALLREDUCE", + forward_comm_size=self.args.seq_length * self.args.micro_batch * 4, + backward_compute_time=compute_time, + backward_comm="NONE", + backward_comm_size=0, + dp_compute_time=compute_time, + dp_comm="NONE", + dp_comm_size=0, + ) + ) + + for i in range(4): + self.workload.append( + Work_Item( + name="optimizer" + str(i + 1), + forward_compute_time=compute_time, + forward_comm="ALLREDUCE", + forward_comm_size=4, + backward_compute_time=compute_time, + backward_comm="NONE", + backward_comm_size=0, + dp_compute_time=compute_time, + dp_comm="NONE", + dp_comm_size=0, + ) + ) + + def dump_file(self, filename): + filename = filename + ".txt" + + pp_comm_value = 2 * self.args.micro_batch * self.args.seq_length * self.args.hidden_size + if self.args.enable_sequence_parallel: + pp_comm_value /= self.args.tensor_model_parallel_size + + pp_comm = ( + f"pp_comm: {pp_comm_value}" + if self.args.pipeline_model_parallel != 1 + else "pp_comm: 0" + ) + with open(filename, "w") as f: + f.write(( + f"HYBRID_TRANSFORMER_FWD_IN_BCKWD model_parallel_NPU_group: {self.args.tensor_model_parallel_size} " + f"ep: {self.args.expert_model_parallel_size} " + f"pp: {self.args.pipeline_model_parallel} " + f"vpp: {self.args.num_layers} " + f"ga: {self.ga_num} all_gpus: {self.args.world_size} " + f"checkpoints: 0 checkpoint_initiates: 0 " + ) + pp_comm + "\n") + + f.write(str(len(self.workload)) + "\n") + for item in self.workload: + f.write( + "\t".join([str(getattr(item, k)) for k in item.__dict__.keys()]) + + "\n" + ) + + +class simAI_MicroTest: + def __init__(self, args): + self.args = args + self.workload = [] + + def _simAI_microtest_convert(self, comm_type): + if comm_type == "all_reduce" or comm_type == "allreduce": + return "ALLREDUCE" + elif comm_type == "all_gather" or comm_type == "allgather": + return "ALLGATHER" + elif comm_type == "reduce_scatter" or comm_type == "reducescatter": + return "REDUCESCATTER" + elif comm_type == "all_to_all" or comm_type == "alltoall": + return "ALLTOALL" + else: + return + + def workload_generator(self): + curr_size = self.args.begin_size + default_compute_time = 1 + while curr_size <= self.args.end_size: + self.workload.append( + Work_Item( + name="micro_test", + forward_compute_time=default_compute_time, + forward_comm="NONE", + forward_comm_size=0, + backward_compute_time=default_compute_time, + backward_comm="NONE", + backward_comm_size=0, + dp_compute_time=default_compute_time, + dp_comm=self._simAI_microtest_convert(self.args.test_comm), + dp_comm_size=curr_size, + process_time=1, + ) + ) + curr_size *= 2 + + def dump_file(self, filename): + filename = filename + ".txt" + with open(filename, "w") as f: + if not self.args.multi_all_reduce_enable: + f.write(f"MICRO" + "\n") + f.write(str(len(self.workload)) + "\n") + for item in self.workload: + f.write( + "\t".join([str(getattr(item, k)) for k in item.__dict__.keys()]) + + "\n" + ) + else: + f.write( + f"HYBRID_TRANSFORMER_FWD_IN_BCKWD model_parallel_NPU_group: {self.args.tensor_model_parallel_size} \ + expert_parallel_npu_group: {self.args.expert_model_parallel_size} pp: {self.args.pipeline_model_parallel} \ + ga: {self.ga_num} all_gpus: {self.args.world_size} checkpoints: 0 checkpoint_initiates: 0" + + "\n" + ) + f.write(str(len(self.workload)) + "\n") + for item in self.workload: + f.write( + "\t".join([str(getattr(item, k)) for k in item.__dict__.keys()]) + + "\n" + ) + + +if __name__ == "__main__": + args = get_params() + print(args) + if args.frame == "DeepSeek": + model = DeepSeekV3Model(args) + elif args.frame == "Qwen3": + model = Qwen3Model(args) + elif args.frame == "Qwen3.5": + model = Qwen3_5Model(args) + else: + model = MegatronModel(args) + result_dir = "results/workload/" + if not os.path.isdir(result_dir): + os.makedirs(result_dir) + filename = f"{args.gpu_type}-{args.model_name}-world_size{args.world_size}-tp{args.tensor_model_parallel_size}-pp{args.pipeline_model_parallel}-ep{args.expert_model_parallel_size}-gbs{args.global_batch}-mbs{args.micro_batch}-seq{args.seq_length}-MOE-{args.moe_enable}-GEMM-{args.moe_grouped_gemm}-flash_attn-{args.use_flash_attn}" + filepath = os.path.join(result_dir, filename) + params = model.parameters() + # work = SIMAI_workload(model, args, GPU_Tensor_core.A100, "gpt13B") + # name_layers = work.workload_generate() + # work.dump_file("test") + print(sum(p.numel() for p in params)) + if args.aiob_enable: + params = model.parameters() + args.model_param = sum(p.numel() for p in params) + if args.comp_filepath == None: + + comp_filepath = get_comp_out(args) + + compute_cache = extract_averages(comp_filepath,args) + else: + print("comp_filepath:", args.comp_filepath) + comp_filepath = args.comp_filepath + compute_cache = extract_averages(comp_filepath,args) + + print("compute_cache = {") + for key, value in compute_cache.items(): + print(f" '{key}' : {value},") + print("}") + work = SIMAI_workload( + model, args,compute_cache + ) + name_layers = work.workload_generate_aiob() + + # set comm_size = 0 for any comm_type == NONE + for i in range(len(work.workload)): + if work.workload[i].forward_comm == "NONE": + work.workload[i].forward_comm_size = 0 + if work.workload[i].backward_comm == "NONE": + work.workload[i].backward_comm_size = 0 + + work.dump_file(filepath) + print("workload save in :", filepath) + # print(args) + else: + + work = SIMAI_workload(model, args, None) + name_layers = work.workload_generate() + work.dump_file(filepath) + print(f"workload save in : {filepath}.txt") diff --git a/aicb/workload_generator/Vidur_workload_generator.py b/aicb/workload_generator/Vidur_workload_generator.py new file mode 100644 index 000000000..5f8c895b9 --- /dev/null +++ b/aicb/workload_generator/Vidur_workload_generator.py @@ -0,0 +1,268 @@ +import sys +import os +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import workload_generator.mocked_model.MockedModel +import workload_generator.mocked_model.inference.MockedDeepSeek as MockedDeepSeek +import workload_generator.mocked_model.inference.MockedQwen3Moe as MockedQwen3Moe +import workload_generator.mocked_model.inference.MockedQwen3Next as MockedQwen3Next +from workload_generator.mocked_model.MockedModel import InferencePhase +from workload_generator.SimAI_inference_workload_generator import _get_aiob_compute_time, _get_model_details, LayerInfo +from utils.utils import CommType, get_params, get_comp_out, extract_inference_averages +import os +from typing import List, Tuple +from collections import deque +import dataclasses +from enum import Enum + +try: + import torch +except ImportError as e: + torch = None + print("Failed to import 'torch'.") +import math +import re +from collections import OrderedDict +''' +可以用字典按键(layer_id, layer_type)聚合，把layer_name丢弃，只累加comp_time与comm_size。示例函数如下： + +def merge_layers(rows, sort=False): + """ + rows: 列表，每项为 [layer_id, layer_name, layer_type, layer_comp_time, layer_comm_size] + 返回: 合并后的列表，每项为 [layer_id, layer_type, sum_comp_time, sum_comm_size] + sort: 是否按 (layer_id, layer_type) 排序输出；默认保持首次出现的顺序 + """ + from collections import OrderedDict + + totals = OrderedDict() # 保持首次出现的顺序 + for layer_id, _, layer_type, comp, comm in rows: + key = (layer_id, layer_type) + if key in totals: + totals[key][0] += comp + totals[key][1] += comm + else: + totals[key] = [comp, comm] + + items = totals.items() + if sort: + items = sorted(items, key=lambda kv: (kv[0][0], kv[0][1])) + + return [[layer_id, layer_type, comp_sum, comm_sum] + for (layer_id, layer_type), (comp_sum, comm_sum) in items] + +示例 +rows = [ + [0, "x", "A", 0, 0], + [0, "y", "A", 0, 0], + [1, "foo", "B", 3.2, 10], + [1, "bar", "B", 1.8, 5], +] +print(merge_layers(rows)) +# 输出: [[0, 'A', 0, 0], [1, 'B', 5.0, 15]] + +如果你用 pandas，也可一行完成： +import pandas as pd +df = pd.DataFrame(rows, columns=["layer_id","layer_name","layer_type","layer_comp_time","layer_comm_size"]) +out = (df + .drop(columns=["layer_name"]) + .groupby(["layer_id","layer_type"], as_index=False) + .sum()) +# out.values.tolist() 得到同样的结果列表 +''' + +class LayerType(Enum): + ATTENTION = "attention" + MLP = "mlp" + MOE = "moe" + +def _get_layer_comm_size(layer_type, tp_size, ep_size): + if layer_type == LayerType.MOE: + return ep_size + else: + return tp_size + +def _extract_layer_type(layer_name): + if "attention" in layer_name: + return LayerType.ATTENTION + elif "mlp" in layer_name: + return LayerType.MLP + elif "moe" in layer_name or "expert" in layer_name: + return LayerType.MOE + else: + return None + + +class VidurWorkload(): + def __init__(self, model, args, compute_cache=None): + self.model = model + self.args = args + self.compute_cache = compute_cache + self.vidur_layers = [] + self.seq_len = args.seq_length + self.tp = args.tensor_model_parallel_size + self.mbs = args.micro_batch + self.batch_size = args.micro_batch # 添加缺失的 batch_size 属性 + self.num_layers = 0 + if args.moe_enable: + self.expert_model_parallel_size = args.expert_model_parallel_size + self.num_experts = args.num_experts + #如果有moe_router_topk则用，否则用num_experts_per_tok + self.topk = args.moe_router_topk if hasattr(args,"moe_router_topk") else args.num_experts_per_tok + + def get_comm_size(self): + phase = getattr(self.args, "phase", InferencePhase.DECODE.value) + if phase == InferencePhase.DECODE.value: + m = self.batch_size + elif phase == InferencePhase.PREFILL.value: + m = self.args.seq_length + tp_comm_size = 2 * m * self.args.hidden_size + ep_combine_size = tp_comm_size * self.topk // self.tp + ep_dispatch_size = ep_combine_size + + if any(s in self.args.frame for s in ('DeepSeek', 'Qwen3')): + # for DeepEP based on https://github.com/parthpower/DeepEP/commit/50aee15f592bc22142eb04b7d718296b19613ae9 + ep_dispatch_size = int(ep_dispatch_size * MockedDeepSeek.FP8_FACTOR) + + return tp_comm_size, ep_dispatch_size, ep_combine_size + + def workload_generate_aiob(self): + vidur_layers_tmp = [] + layers = _get_model_details(self.model) + for layer in layers: + # print(f'Layer {layer.layer_id} {layer.layer_name}') + layer_type = _extract_layer_type(layer.layer_name) + comp_time = _get_aiob_compute_time( + self.compute_cache, "forward", layer.layer_name + ) + vidur_layers_tmp.append( + [layer.layer_id, + layer.layer_name, + layer_type, + comp_time, + 0] + ) + totals = OrderedDict() + for layer_id, _, layer_type, comp, comm in vidur_layers_tmp: + key = (layer_id, layer_type) + if key in totals: + totals[key][0] += comp + totals[key][1] += comm + else: + totals[key] = [comp, comm] + + items = totals.items() + + self.vidur_layers = [[layer_id, layer_type, comp_sum, comm_sum] + for (layer_id, layer_type), (comp_sum, comm_sum) in items] + + # Comm part + tp_comm_size, ep_dispatch_size, ep_combine_size = self.get_comm_size() + tp_comm_size = tp_comm_size if self.tp > 1 else 0 + ep_comm_size = ep_dispatch_size + ep_combine_size + for vidur_layer in self.vidur_layers: + comm_size = _get_layer_comm_size(vidur_layer[1], tp_comm_size, ep_comm_size) + vidur_layer[3] = comm_size + + def dump_file(self, filename): + filename = filename + ".csv" + with open(filename, "w") as f: + f.write("layer_id\tlayer_name\tcomp_time\tcomm_size\n") + # for layer_name, layer_info in self.workload.items(): + # if layer_info.layer_comp_time == 0: + # continue + # f.write(f"{layer_name}\t{layer_info.layer_comp_time}\t{layer_info.layer_comm_size}\n") + for vidur_layer in self.vidur_layers: + f.write(f'{vidur_layer[0]}\t{vidur_layer[1].value}\t{vidur_layer[2]}\t{vidur_layer[3]}\n') + +if __name__ == "__main__": + import sys + import argparse + from utils.utils import get_params + + # Parse command line arguments + parser = argparse.ArgumentParser(description="Generate inference workload for AI models") + parser.add_argument("model_name", help="Model name (e.g., DeepSeek-671B, Qwen3-Moe-235B, Qwen3-Next-80B)") + parser.add_argument("config_file", nargs="?", help="Path to JSON config file") + + # Add arguments that has default value + parser.add_argument("--aiob_enable", action="store_true", default=False, help="Enable AIOB") + parser.add_argument("--aiob_forward_loops", type=int, default=1, help="Number of AIOB forward loops") + parser.add_argument("--seq_length", type=int, default=1, help="Sequence length") + parser.add_argument("--micro_batch", type=int, default=1, help="Micro batch size") + parser.add_argument("--world_size", type=int, default=1, help="World size") + parser.add_argument("--tensor_model_parallel_size", default=1, type=int, help="Tensor model parallel size") + parser.add_argument("--expert_model_parallel_size", default=1, type=int, help="Expert model parallel size") + parser.add_argument("--pipeline_model_parallel", default=1, type=int, help="Pipeline model parallel size") + parser.add_argument("--moe_enable", default=True, action="store_true", help="Enable MoE") + parser.add_argument("--result_dir", default="results/workload/", help="Result directory") + parser.add_argument("--phase", + choices=[InferencePhase.DECODE.value, InferencePhase.PREFILL.value], + default=InferencePhase.DECODE.value, help="Inference phase") + + args = parser.parse_args() + + model_name = args.model_name + config_file = args.config_file + + if "Qwen3-Moe" in model_name: + args = MockedQwen3Moe.Qwen3MoeParams(config_file, args) + model = MockedQwen3Moe.Qwen3MoeModel(args) + elif "Qwen3-Next" in model_name: + args = MockedQwen3Next.Qwen3NextParams(config_file, args) + model = MockedQwen3Next.Qwen3NextModel(args) + elif "DeepSeek" in model_name: + args = MockedDeepSeek.DeepSeekParams(config_file, args) + model = MockedDeepSeek.DeepSeekModel(args) + else: + print(f"Invalid model name: {model_name}") + sys.exit(1) + + phase = getattr(args, "phase", InferencePhase.DECODE.value) + result_dir = args.result_dir + if not os.path.isdir(result_dir): + os.makedirs(result_dir) + filename = f"vidur-{args.model_name}-world_size{args.world_size}-tp{args.tensor_model_parallel_size}-pp{args.pipeline_model_parallel}-ep{args.expert_model_parallel_size}-bs{args.micro_batch}-seq{args.seq_length}-{phase}" + + if args.aiob_enable: + if "Qwen3-Moe" in model_name: + import workload_generator.mocked_model.inference.AiobQwen3Moe as AiobQwen3Moe + aiob_model = AiobQwen3Moe.Qwen3MoeModel(args) + aiob_output_filepath = aiob_model() + elif "Qwen3-Next" in model_name: + import workload_generator.mocked_model.inference.AiobQwen3Next as AiobQwen3Next + aiob_model = AiobQwen3Next.Qwen3NextModel(args) + aiob_output_filepath = aiob_model() + elif "DeepSeek" in model_name: + import workload_generator.mocked_model.inference.AiobDeepSeek as AiobDeepSeek + aiob_model = AiobDeepSeek.DeepSeekModel(args) + aiob_output_filepath = aiob_model() + else: + print(f"Invalid model name: {model_name}") + sys.exit(1) + + else: + aiob_dir = "results/aiob_outputs" + aiob_output_filename = f"{args.model_name}-world_size{args.world_size}-tp{args.tensor_model_parallel_size}-pp{args.pipeline_model_parallel}-ep{args.expert_model_parallel_size}-bpg{args.micro_batch}-seq{args.seq_length}-{phase}.txt" + aiob_output_filepath = os.path.join(aiob_dir,aiob_output_filename) + if not os.path.exists(aiob_output_filepath): + print(f"aiob not enabled, and {aiob_output_filepath} not found. Using default compute time.") + aiob_output_filepath = "" + else: + print(f"aiob not enabled, using existing file {aiob_output_filepath}.") + compute_cache = extract_inference_averages(aiob_output_filepath,args) + print("compute_cache = {") + for key, value in compute_cache.items(): + print(f" '{key}' : {value},") + print("}") + + work = VidurWorkload( + model, args, compute_cache + ) + work.workload_generate_aiob() + filepath = os.path.join(result_dir, filename) + work.dump_file(filepath) + print("workload save in :", filepath) + print("Finish Model initialization") + + # TODO: 实现Vidur工作负载生成和保存逻辑 + # print("VidurWorkload class initialized. Implementation pending.") \ No newline at end of file diff --git a/aicb/workload_generator/__init__.py b/aicb/workload_generator/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/aicb/workload_generator/__init__.py @@ -0,0 +1 @@ + diff --git a/aicb/workload_generator/analysis_pytorch_trace.py b/aicb/workload_generator/analysis_pytorch_trace.py new file mode 100644 index 000000000..6d6ddbbcf --- /dev/null +++ b/aicb/workload_generator/analysis_pytorch_trace.py @@ -0,0 +1,72 @@ +""" +Copyright (c) 2021, Alibaba Group; +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import json +from utils.utils import CommGroup, CommType +from log_analyzer.log import LogItem +from workload_generator.mocked_model.MockedModel import MockedModel +from workload_generator.workload_generator import WorkloadGenerator + +comm_node = {} +workload = [] + + +class Pytorch_trace_analyer(WorkloadGenerator): + def __init__(self, args, model, filename): + super().__init__(args, model) + self.name = "pytorch_trace" + self.filename = filename + + def init(self): + pass + + def string2comm_type(self, s): + if "all_gather" in s or "_all_gather_base" in s or "_allgather_base" in s: + return CommType.all_gather + if "reduce_scatter" in s or "_reduce_scatter_base" in s: + return CommType.reduce_scatter + if "all_reduce" in s: + return CommType.all_reduce + if "broadcast" in s: + return CommType.broadcast + if "barrier" in s: + return CommType.barrier + if "reduce" in s: + return CommType.reduce + else: + print(f"can not convert {s} to any comm type") + exit(0) + + def step(self): + item = LogItem() + with open(self.filename) as f: + data = json.load(f) + + nodes_list = data["nodes"] + for node in nodes_list: + if node["name"].startswith("nccl:"): + name = node["name"].split(":")[1] + comm_type = self.string2comm_type(name) + item.comm_type = comm_type + # TODO: set group in default dp_group , get group in trace info + item.comm_group = CommGroup.dp_group + input = node.get("inputs") + item.msg_size = input[0][3] + item.item_size = input[0][4] + self.workload.append(item) + + +# if __name__ == "__main__": +# a = parse_pytorch_trace_log("llama7b_zero3_rank8.json") +# json_parse("test_json.json") +# panda_parse("test_json.json") diff --git a/aicb/workload_generator/generate_collective_test.py b/aicb/workload_generator/generate_collective_test.py new file mode 100644 index 000000000..f91de3c57 --- /dev/null +++ b/aicb/workload_generator/generate_collective_test.py @@ -0,0 +1,80 @@ +""" +Copyright (c) 2021, Alibaba Group; +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from utils.utils import CommType, CommGroup, get_params, WorkloadWriter +from log_analyzer.log import LogItem, Workload +from workload_generator.workload_generator import WorkloadGenerator + + +class Collective_Test(WorkloadGenerator): + def __init__(self, args, model): + super().__init__(args, model) + self.args = args + self.name = "collective_test" + + def init(self): + iter_num = self.args.iter_num + for i in range(iter_num): + # for warmup + self.workload.append( + LogItem( + comm_type=CommType.get_comm_type(self.args.test_comm), + comm_group=CommGroup.dp_group, + comm_group_size=self.args.dp_num, + msg_size=self.args.begin_size, + stage="warmup", + ) + ) + + def step(self): + test_comm = CommType.get_comm_type(self.args.test_comm) + begin_size = self.args.begin_size + end_size = self.args.end_size + curr_size = begin_size + iter_num = self.args.iter_num + multi_all_reduce_enable = self.args.multi_all_reduce_enable + + while curr_size <= end_size: + # self.workload.append(LogItem(comm_type=CommType.epoch_end)) + if not multi_all_reduce_enable: + for i in range(iter_num): + self.workload.append( + LogItem( + comm_type=test_comm, + comm_group=CommGroup.dp_group, + comm_group_size=self.args.dp_num, + msg_size=curr_size, + stage="test_step", + ) + ) + curr_size *= 2 + else: + for i in range(iter_num): + self.workload.append( + LogItem( + comm_type=test_comm, + comm_group=CommGroup.pp_group, + comm_group_size=self.args.pipeline_model_parallel, + msg_size=curr_size, + stage="test_step", + ) + ) + curr_size *= 2 + + +if __name__ == "__main__": + args = get_params() + workload_generator = Collective_Test(args, None) + workload = workload_generator() + filename = "multi_all_reduce.csv" + workload.dump(filename) diff --git a/aicb/workload_generator/generate_deepspeed_stage1_2_workload.py b/aicb/workload_generator/generate_deepspeed_stage1_2_workload.py new file mode 100644 index 000000000..2bcb2e438 --- /dev/null +++ b/aicb/workload_generator/generate_deepspeed_stage1_2_workload.py @@ -0,0 +1,267 @@ +""" +Copyright (c) 2021, Alibaba Group; +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +"""example of running zero1/2 on llama-13B +python -m workload_generator.deepspeed_stage1_workload \ + --stage=1 --world_size=624 --global_batch=624 \ + --num_layers=40 --epoch_num=2 --hidden_size=5120 --ffn_hidden_size=13696 \ + --reduce_bucket_size=2000000000 --allgather_bucket_size=2000000000 + +python -m workload_generator.deepspeed_stage1_2_workload \ + --stage=2 --world_size=256 --global_batch=1024 --num_attention_heads=40 \ + --num_layers=40 --epoch_num=100 --hidden_size=5120 --ffn_hidden_size=13824 \ + --reduce_bucket_size=26214400 --allgather_bucket_size=500000000 --contiguous_gradients +""" + +import math +from workload_generator.mocked_model.training.MockedDeepspeed import DeepspeedForCausalLM +from workload_generator.mocked_model.MockedModel import MockedModel +from workload_generator.workload_generator import WorkloadGenerator +from utils.utils import CommGroup, CommType, get_params, WorkloadWriter +from log_analyzer.log import LogItem + + +class DeepSpeedStage1(WorkloadGenerator): + """workload generator for deepspeed engine setup + mock comm behavior of DeepSpeedEngine.__init__ + """ + + def __init__(self, args, model) -> None: + super().__init__(args, model) + self.name = "deepspeed_stage1" + self.compute_enable = args.computation_enable + self.batch_size = args.micro_batch + self.seq_len = args.seq_length + self.reduce_bucket, self.num_in_reduce_bucket, self.max_reduce_bucket_size = ( + [], + 0, + args.reduce_bucket_size, + ) + self.allgather_bucket_size = args.allgather_bucket_size + self.amp_enabled = args.amp_enabled + self.dp_world_size = args.dp_num + self.elem_size = 2 + self.all_params = list(self.model.parameters()) + + def init(self): + if not self.amp_enabled: + for param in self.model.parameters(): + self.workload.append( + LogItem( + comm_type=CommType.broadcast, + comm_group=CommGroup.dp_group, + comm_group_size=self.dp_world_size, + msg_size=param.msg_size(), + stage="init", + ) + ) + + self.workload.append( + LogItem( + comm_type=CommType.barrier, + comm_group=CommGroup.all, + comm_group_size=self.dp_world_size, + msg_size=param.msg_size(), + stage="init.__init__", + ) + ) + + def forward(self): + if self.compute_enable: + self.all_params = list(self.model.parameters()) + for param in self.all_params: + if param.get_shape()[-1] != 1: + self.workload.append( + LogItem( + comm_type=CommType.computation, + msg_size=( + (self.batch_size, self.seq_len, param.get_shape()[0]), + (param.get_shape()[0], param.get_shape()[1]), + ), + stage="forward.computation", + ) + ) + + def _reduce_ipg_grads(self): + self.workload.append( + LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.dp_group, + comm_group_size=self.dp_world_size, + msg_size=self.num_in_reduce_bucket * self.elem_size, + stage=f"{self.current_op}.allreduce_bucket", + ) + ) + self.reduce_bucket, self.num_in_reduce_bucket = [], 0 + + def backward(self): + self.current_op = "backward" + for param in self.all_params[::-1]: + if param.numel() + self.num_in_reduce_bucket > self.max_reduce_bucket_size: + self._reduce_ipg_grads() + self.reduce_bucket.append(param) + self.num_in_reduce_bucket += param.numel() + if self.compute_enable: + if param.get_shape()[-1] != 1: + self.workload.append( + LogItem( + comm_type=CommType.computation, + msg_size=( + (self.batch_size, self.seq_len, param.get_shape()[0]), + (param.get_shape()[0], param.get_shape()[1]), + ), + stage=f"{self.current_op}.computation", + ) + ) + self.workload.append( + LogItem( + comm_type=CommType.computation, + msg_size=( + (param.get_shape()[0], self.batch_size * self.seq_len), + (self.batch_size * self.seq_len, param.get_shape()[1]), + ), + stage=f"{self.current_op}.computation", + ) + ) + + def step(self): + self.current_op = "step" + self._reduce_ipg_grads() + + self.workload.append( + LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.dp_group, + comm_group_size=self.dp_world_size, + msg_size=1, + stage=f"{self.current_op}.has_overflow", + ) + ) + num_params = sum([param.numel() for param in self.model.parameters()]) + num_shards = max(num_params // self.allgather_bucket_size, 1) + shard_size = num_params // num_shards + + for i in range(num_shards): + num_elements = ( + num_params - i * shard_size if i == (num_shards - 1) else shard_size + ) + padding_size = ( + (self.dp_world_size - num_elements % self.dp_world_size) + if num_elements % self.dp_world_size + else 0 + ) + num_elements = num_elements + padding_size + self.workload.append( + LogItem( + comm_type=CommType.all_gather, + comm_group=CommGroup.dp_group, + comm_group_size=self.dp_world_size, + msg_size=num_elements * self.elem_size, + stage=f"{self.current_op}.all_gather_dp_groups", + ) + ) + + +class DeepSpeedStage2(DeepSpeedStage1): + def __init__(self, args, model) -> None: + super().__init__(args, model) + self.name = "deepspeed_stage2" + + self.param_range_map = self.build_model_gbuf_param_range_map( + model, self.dp_world_size + ) + + def build_model_gbuf_param_range_map(self, model: MockedModel, dp_world_size: int): + gbuf_size = sum([param.numel() for param in model.parameters()]) + + gbuf_partition_size = int(math.ceil(gbuf_size / dp_world_size)) + gbuf_world_all_ranges = [] + for r in range(dp_world_size): + gbuf_world_start = r * gbuf_partition_size + gbuf_world_end = min(gbuf_size, gbuf_world_start + gbuf_partition_size) + gbuf_world_all_ranges.append((gbuf_world_start, gbuf_world_end)) + + start_idx, r = 0, 0 + gbuf_world_start, gbuf_world_end = gbuf_world_all_ranges[r] + # record each param should be reduced to which rank(s) + # param_id: int -> List[(rank: int, param_start_idx: int, param_end_idx: int)] + param_range_map = {} + for param in self.all_params: + # current param in [start_idx, end_idx) range of gbuf + param_id = id(param) + param_range_map[param_id] = [] + end_idx = start_idx + param.numel() + + # current rank is in change of [gbuf_world_start, gbuf_world_end) of gbuf + param_start_idx = start_idx + # if current rank cannot fully cover this param, move to next rank + while gbuf_world_end < end_idx: + param_range_map[param_id].append((r, param_start_idx, gbuf_world_end)) + param_start_idx = gbuf_world_end + r += 1 + gbuf_world_start, gbuf_world_end = gbuf_world_all_ranges[r] + param_range_map[param_id].append((r, param_start_idx, end_idx)) + + # for next param + start_idx = end_idx + return param_range_map + + def _reduce_ipg_grads(self): + if not self.args.contiguous_gradients: + super()._reduce_ipg_grads() + return + + rank_start_end_idx = [[-1, -1, -1]] + for param in self.reduce_bucket[::-1]: + for rank, start_idx, end_idx in self.param_range_map[id(param)]: + if rank == rank_start_end_idx[-1][0]: + if rank_start_end_idx[-1][-1] != start_idx: + print(f"WARNNING {rank_start_end_idx[-1]} - {start_idx}") + rank_start_end_idx[-1][-1] = end_idx + else: + rank_start_end_idx.append([rank, start_idx, end_idx]) + + for rank, start_idx, end_idx in rank_start_end_idx[1:]: + self.workload.append( + LogItem( + comm_type=CommType.reduce, + comm_group=CommGroup.dp_group, + msg_size=(end_idx - start_idx) * self.elem_size, + comm_group_size=self.dp_world_size, + stage=f"{self.current_op}.average_tensor", + dst=rank, + ) + ) + self.reduce_bucket, self.num_in_reduce_bucket = [], 0 + + +if __name__ == "__main__": + args = get_params() + print(args.__dict__) + model = DeepspeedForCausalLM(args) + if args.stage == 1: + workload_generator = DeepSpeedStage1(args, model) + filename = f"{workload_generator.name}_{args.model_name}_sp_{args.enable_sequence_parallel}_iteration_{args.epoch_num}_computationEnable_{args.computation_enable}_{args.world_size}n.csv" + else: + workload_generator = DeepSpeedStage2(args, model) + filename = f"{workload_generator.name}_{args.model_name}_sp_{args.enable_sequence_parallel}_iteration_{args.epoch_num}_computationEnable_{args.computation_enable}_{args.world_size}n.csv" + workload = workload_generator() + workload.dump(filename) + if args.enable_visual: + try: + from visualize.generate import visualize_output + base_name = filename.split(".")[0] + visualize_output(f"./results/mocked_workload/{base_name}_workload.csv",True) + except ImportError: + print("visualize_output is not available because required library is not found") + # WorkloadWriter.write_workload(workload, args, filename) diff --git a/aicb/workload_generator/generate_deepspeed_stage3_workload.py b/aicb/workload_generator/generate_deepspeed_stage3_workload.py new file mode 100644 index 000000000..d40e3be4a --- /dev/null +++ b/aicb/workload_generator/generate_deepspeed_stage3_workload.py @@ -0,0 +1,325 @@ +""" +Copyright (c) 2021, Alibaba Group; +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +""" +python -m workload_generator.generate_deepspeed_stage3_workload \ + --stage=3 --world_size=256 --global_batch=1024 --num_attention_heads=40 \ + --num_layers=40 --epoch_num=100 --hidden_size=5120 --ffn_hidden_size=13824 \ + --reduce_bucket_size=26214400 --allgather_bucket_size=500000000 --contiguous_gradients +""" + +from workload_generator.mocked_model.training.MockedDeepspeed import DeepspeedForCausalLM +from workload_generator.mocked_model.MockedModel import MockedModel +from workload_generator.workload_generator import WorkloadGenerator +from utils.utils import CommGroup, CommType, get_params, WorkloadWriter +from collections import deque, defaultdict +from log_analyzer.log import LogItem + + +class DeepSpeedStage3(WorkloadGenerator): + """workload generator for deepspeed engine setup + mock comm behavior of DeepSpeedEngine.__init__ + """ + + def __init__(self, args, model) -> None: + super().__init__(args, model) + self.name = "deepspeed_stage3" + self.amp_enabled = args.amp_enabled + self.dp_world_size = args.dp_num + self.batch_size = args.micro_batch + self.seq_len = args.seq_length + self.compute_enable = args.computation_enable + self.reduce_bucket, self.reduce_bucket_size = 0, args.reduce_bucket_size + self.prefetch_bucket_size = args.prefetch_bucket_size + self.max_live_parameters, self.current_live_parameters = ( + args.max_live_parameters, + 0, + ) + self.stage, self._param_queue, self.all_params = ( + "init", + deque(), + list(self.model.parameters()), + ) + self.__param_order = [ + (param, step_id) + for step_id, param in enumerate(self.all_params + self.all_params[::-1]) + ] + self.__most_recent_step_id_param_fetched_for = defaultdict(lambda: -1) + self._mark_persistent_parameters( + args.param_persistence_threshold, args.model_persistence_threshold + ) + + def _mark_persistent_parameters(self, param_threshold, model_threshold): + self.persistent_params = [] + total_persistent_parameters = 0 + count = 0 + for param in self.model.parameters(): + param.id = count # is also the step id + count += 1 + param.ds_persist = False + param.has_been_allgather = False + if param.numel() + total_persistent_parameters > model_threshold: + continue + if param.numel() <= param_threshold: + param.ds_persist = True + self.persistent_params.append(param) + total_persistent_parameters += param.numel() + + def init(self): + if not self.amp_enabled: + for param in self.model.parameters(): + self.workload.append( + LogItem( + comm_type=CommType.broadcast, + comm_group=CommGroup.dp_group, + comm_group_size=self.dp_world_size, + msg_size=param.msg_size(), + stage="init._broadcast_model", + src=0, + ) + ) + + self.workload.append( + LogItem( + comm_type=CommType.barrier, + comm_group=CommGroup.all, + comm_group_size=self.dp_world_size, + msg_size=0, + stage="init._create_fp16_partitions_with_defragmentation", + ) + ) + + for _ in range(2): + self.workload.append( + LogItem( + comm_type=CommType.barrier, + comm_group=CommGroup.all, + comm_group_size=self.dp_world_size, + msg_size=0, + stage="init._setup_for_real_optimizer", + ) + ) + + for param in self.model.parameters(): + self.workload.append( + LogItem( + comm_type=CommType.all_gather, + comm_group=CommGroup.dp_group, + comm_group_size=self.dp_world_size, + msg_size=param.msg_size(), + stage="init._allgather_params", + ) + ) + + def _compute_for_param(self, param): + if self.stage == "forward": + if param.get_shape()[-1] != 1: + self.workload.append( + LogItem( + comm_type=CommType.computation, + msg_size=( + (self.batch_size, self.seq_len, param.get_shape()[0]), + (param.get_shape()[0], param.get_shape()[1]), + ), + stage=f"{self.stage}.computation", + ) + ) + if self.stage == "backward": + # input grad + if param.get_shape()[-1] != 1: + self.workload.append( + LogItem( + comm_type=CommType.computation, + msg_size=( + (self.batch_size, self.seq_len, param.get_shape()[0]), + (param.get_shape()[0], param.get_shape()[1]), + ), + stage=f"{self.stage}.computation", + ) + ) + + # weight grad + self.workload.append( + LogItem( + comm_type=CommType.computation, + msg_size=( + (param.get_shape()[0], self.batch_size * self.seq_len), + (self.batch_size * self.seq_len, param.get_shape()[1]), + ), + ) + ) + + def _gather_param_directly(self, param): + if not param.has_been_allgather: + self.workload.append( + LogItem( + comm_type=CommType.all_gather, + comm_group=CommGroup.dp_group, + comm_group_size=self.dp_world_size, + msg_size=param.msg_size(), + stage=f"{self.stage}.allgather_fn", + ) + ) + param.has_been_allgather = True + self.current_live_parameters += param.numel() + if self.compute_enable: + self._compute_for_param(param) + + def _gather_param_prefetch(self, param, step_id): + prefetch_bucket, prefetch_bucket_size = [], 0 + if not param.has_been_allgather: + prefetch_bucket.append(param) + prefetch_bucket_size += param.numel() + future_param, future_step_id = self._param_queue.popleft() + if future_param != param: + print( + f"WARNING: expected {param.__dict__, step_id} but got {future_param.__dict__, future_step_id}" + ) + param.has_been_allgather = True + self.current_live_parameters += param.numel() + + while ( + self._param_queue + and prefetch_bucket_size < self.prefetch_bucket_size + and self.current_live_parameters < self.max_live_parameters + ): + future_param, step_id = self._param_queue.popleft() + self.__most_recent_step_id_param_fetched_for[future_param.id] = max( + step_id, self.__most_recent_step_id_param_fetched_for[future_param.id] + ) + if future_param.has_been_allgather: + continue + prefetch_bucket.append(future_param) + future_param.has_been_allgather = True + self.current_live_parameters += future_param.numel() + prefetch_bucket_size += future_param.numel() + + if prefetch_bucket: + self.workload.append( + LogItem( + comm_type=CommType.all_gather, + comm_group=CommGroup.dp_group, + comm_group_size=self.dp_world_size, + msg_size=sum(param.msg_size() for param in prefetch_bucket), + stage=f"{self.stage}.allgather_fn", + ) + ) + if self.compute_enable: + for param in prefetch_bucket: + self._compute_for_param(param) + + def _partition_param(self, param, step_id): + if len(self._param_queue) == 0: + # 这里会错误的释放一些ds_persist的参数，但是不影响整体的模拟 + param.has_been_allgather = False + self.current_live_parameters -= param.numel() + return + if param.ds_persist: + return + # 这里说明之后马上还会用到这个param + if self.__most_recent_step_id_param_fetched_for[param.id] > step_id: + return + param.has_been_allgather = False + self.current_live_parameters -= param.numel() + + def forward(self): + self.stage = "forward" + for i, param in enumerate(self.all_params): + if len(self._param_queue) == 0: + self._gather_param_directly(param) + else: + self._gather_param_prefetch(param, i) + self._partition_param(param, i) + + def _reduce_param_with_bucket(self, param): + if param.numel() + self.reduce_bucket > self.reduce_bucket_size: + self.workload.append( + LogItem( + comm_type=CommType.reduce_scatter, + comm_group=CommGroup.dp_group, + comm_group_size=self.dp_world_size, + msg_size=self.reduce_bucket * param.elem_size(), + stage=f"{self.stage}.reduce_scatter_fn", + ) + ) + self.reduce_bucket = param.numel() + else: + self.reduce_bucket += param.numel() + + def backward(self): + self.stage = "backward" + for i, param in enumerate(self.all_params[::-1]): + if len(self._param_queue) == 0: + self._gather_param_directly(param) + else: + self._gather_param_prefetch(param, i) + self._partition_param(param, i + len(self.all_params)) + self._reduce_param_with_bucket(param) + self._param_queue = deque(self.__param_order) + self.__most_recent_step_id_param_fetched_for = defaultdict(lambda: -1) + + def step(self): + self.stage = "step" + self.workload.append( + LogItem( + comm_type=CommType.reduce_scatter, + comm_group=CommGroup.dp_group, + comm_group_size=self.dp_world_size, + msg_size=self.reduce_bucket * 2, + stage=f"{self.stage}.reduce_scatter_fn", + ) + ) + self.reduce_bucket = 0 + + self.workload.append( + LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.dp_group, + comm_group_size=self.dp_world_size, + msg_size=1, + stage=f"{self.stage}.has_overflow", + ) + ) + self.workload.append( + LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.dp_group, + comm_group_size=self.dp_world_size, + msg_size=8, + stage=f"{self.stage}.get_grad_norm_direct", + ) + ) + + for param in self.model.parameters(): + param.has_been_allgather = False + self.current_live_parameters = 0 + + for param in self.persistent_params: + self._gather_param_directly(param) + + +if __name__ == "__main__": + args = get_params() + model = DeepspeedForCausalLM(args) + workload_generator = DeepSpeedStage3(args, model) + workload = workload_generator() + filename = f"{workload_generator.name}_{args.model_name}_sp_{args.enable_sequence_parallel}_iteration_{args.epoch_num}_computationEnable_{args.computation_enable}_{args.world_size}n.csv" + workload.dump(filename) + if args.enable_visual: + try: + from visualize.generate import visualize_output + base_name = filename.split(".")[0] + visualize_output(f"./results/mocked_workload/{base_name}_workload.csv",True) + except ImportError: + print("visualize_output is not available because required library is not found") + # WorkloadWriter.write_workload(workload, args, filename) diff --git a/aicb/workload_generator/generate_ds_trace_replay_workload.py b/aicb/workload_generator/generate_ds_trace_replay_workload.py new file mode 100644 index 000000000..6f1250930 --- /dev/null +++ b/aicb/workload_generator/generate_ds_trace_replay_workload.py @@ -0,0 +1,56 @@ +""" +Copyright (c) 2021, Alibaba Group; +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import re +from utils.utils import CommType, CommGroup, WorkloadWriter, get_params + + +class TraceParser: + def __init__(self, input_file): + self.input_file = input_file + self.comm_workload = [] + + def prase_trace(self): + + pattern = r"comm op: (.*?) \|.*?msg size: (.*?) \|.*?algbw \(Gbps\): (.*?) " + + with open(self.input_file, "r") as file: + for line in file: + match = re.search(pattern, line) + if match: + op = match.group(1) + comm_op = CommType.get_comm_type(op) + msg_size = match.group(2) + self.comm_workload.append( + { + "operation": "trace", + "comm_type": comm_op, + "msg_size": msg_size, + "comm_group": CommGroup.dp_group, + "bw": match.group(3), + } + ) + + def get_trace_workload(self): + return self.comm_workload + + +if __name__ == "__main__": + args = get_params() + output_file = "model_workload/deepspeed_trace.csv" + paser = TraceParser( + "llama-7b-ga8-seq2048-bs3_dlcfw77d07c87pho-master-0_2023-07-05 21_48_38.txt" + ) + paser.prase_trace() + workload = paser.get_trace_workload() + WorkloadWriter().write_workload(workload, args, output_file) diff --git a/aicb/workload_generator/generate_megatron_workload.py b/aicb/workload_generator/generate_megatron_workload.py new file mode 100644 index 000000000..e0770e661 --- /dev/null +++ b/aicb/workload_generator/generate_megatron_workload.py @@ -0,0 +1,466 @@ +""" +Copyright (c) 2021, Alibaba Group; +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +#!/bin/python +"""example of running megatron on gpt-7B +python -m workload_generator.megatron_workload \ + --frame=Megatron --world_size=16 --tensor_model_parallel_size=8 --pipeline_model_parallel=1 --global_batch=64 --micro_batch=2 \ + --num_layers=32 --seq_length=2048 --hidden_size=4096 --epoch_num=2 --use-distributed-optimizer --enable_sequence_parallel +""" +from utils.utils import CommGroup, CommType, get_params, WorkloadWriter, num_parameters_to_bytes +from workload_generator.workload_generator import WorkloadGenerator +from workload_generator.mocked_model.training.MockedMegatron import MegatronModel +from workload_generator.mocked_model.training.MockedDeepSeek import DeepSeekV3Model +from workload_generator.mocked_model.training.MockedQwen3 import Qwen3Model +from workload_generator.mocked_model.training.MockedQwen3_5 import Qwen3_5Model +from log_analyzer.log import LogItem + + +class MegatronWorkload(WorkloadGenerator): + def __init__(self, args, model): + super().__init__(args, model) + self.name = "megatron" + self.args = args + self.tp_is_enable = True if args.tensor_model_parallel_size > 1 else False + # print(f"total params: {self._get_total_params()}") + + def _get_total_params(self): + total_params = 0 + for param in self.model.parameters(): + total_params += param.numel() + return total_params + + def _get_layernorm_params(self): + total_params = 0 + for param in self.model.parameters(): + if getattr(param, "sequence_parallel", False): + total_params += param.numel() + return total_params + + def init(self): + args = self.args + self.workload.append( + LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.dp_group, + comm_group_size=self.args.dp_num, + msg_size=1 * 8, + stage="init.model_setup", + ) + ) + for _ in range(3): + self.workload.append( + LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.dp_group, + comm_group_size=self.args.dp_num, + msg_size=1 * 8, + stage="init.model_setup", + ) + ) + if args.pipeline_model_parallel > 1: + self.workload.append( + LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.pp_group, + comm_group_size=self.args.pipeline_model_parallel, + msg_size=1 * 8, + stage="init.model_setup", + ) + ) + # time + self.workload.append( + LogItem( + comm_type=CommType.all_gather, + comm_group=CommGroup.dp_group, + comm_group_size=self.args.dp_num, + msg_size=4 * 8, + stage="init.model_setup", + ) + ) + + self.workload.append( + LogItem( + comm_type=CommType.broadcast, + comm_group=CommGroup.tp_group, + comm_group_size=self.args.tensor_model_parallel_size, + msg_size=3 * 8, + stage="init.model_setup", + src=0, + ) + ) + + if args.pp_rank == args.pipeline_model_parallel - 1 and args.pipeline_model_parallel > 1: + for p in self.model.embedding.parameters(): + self.workload.append( + LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.tp_group, + comm_group_size=self.args.tensor_model_parallel_size, + msg_size=p.msg_size(), + stage="init.model_setup", + ) + ) + # time + self.workload.append( + LogItem( + comm_type=CommType.all_gather, + comm_group=CommGroup.dp_group, + comm_group_size=self.args.dp_num, + msg_size=8 * 8, + stage="init.model_setup", + ) + ) + + def get_pp_rank(self, rank, world_size, pp_size): + ranks_per_pp_group = world_size // pp_size + pp_rank = rank // ranks_per_pp_group + return pp_rank + + def with_pipeline_forward_backward(self): + args = self.args + if args.workload_only: + rank = 0 + else: + import torch + rank = torch.distributed.get_rank() + world_size = args.world_size + pp_rank = self.get_pp_rank(rank, world_size, args.pipeline_model_parallel) + pp_num_warmup_microbatches = min( + args.pipeline_model_parallel - pp_rank - 1, args.num_microbatches + ) + num_microbatches_remaining = args.num_microbatches - pp_num_warmup_microbatches + temp = self.model.forward() + # forward_comm = self._get_comm_op(temp) + + for _ in range(pp_num_warmup_microbatches): + if pp_rank != 0: + # recv_prev + self.workload.append( + LogItem( + comm_type=CommType.irecv, + comm_group=CommGroup.pp_group, + comm_group_size=1, + msg_size=2 + * (args.hidden_size * args.seq_length * args.micro_batch), + stage="forward_step", + additional="recv_prev", + ) + ) + self.workload.append( + LogItem( + comm_type=CommType.broadcast, + comm_group=CommGroup.tp_group, + comm_group_size=self.args.tensor_model_parallel_size, + msg_size=5 * 8, + stage="forward_step", + src=0, + ) + ) + self.workload.append( + LogItem( + comm_type=CommType.broadcast, + comm_group=CommGroup.tp_group, + comm_group_size=self.args.tensor_model_parallel_size, + msg_size=8 * (args.world_size + args.seq_length * args.micro_batch), + stage="forward_step", + src=0, + ) + ) + + # for item in forward_comm: + self.workload.extend(self.model.forward()) + + if pp_rank != args.pipeline_model_parallel - 1: + # send_next + self.workload.append( + LogItem( + comm_type=CommType.isend, + comm_group=CommGroup.pp_group, + comm_group_size=1, + msg_size=2 + * (args.hidden_size * args.seq_length * args.micro_batch), + stage="forward_step", + additional="send_next", + ) + ) + # recv prev + if num_microbatches_remaining > 0 and pp_rank != 0: + self.workload.append( + LogItem( + comm_type=CommType.irecv, + comm_group=CommGroup.pp_group, + comm_group_size=1, + msg_size=2 + * (args.hidden_size * args.seq_length * args.micro_batch), + stage="forward_step", + additional="recv_prev", + ) + ) + + for i in range(num_microbatches_remaining): + last_iter = i == (num_microbatches_remaining - 1) + self.workload.append( + LogItem( + comm_type=CommType.broadcast, + comm_group=CommGroup.tp_group, + comm_group_size=self.args.tensor_model_parallel_size, + msg_size=5 * 8, + stage="forward_step", + src=0, + ) + ) + self.workload.append( + LogItem( + comm_type=CommType.broadcast, + comm_group=CommGroup.tp_group, + comm_group_size=self.args.tensor_model_parallel_size, + msg_size=8 * (args.world_size + args.seq_length * args.micro_batch), + stage="forward_step", + src=0, + ) + ) + + self.workload.extend(self.model.forward()) + if pp_rank != args.pipeline_model_parallel - 1: + # recv next + self.workload.append( + LogItem( + comm_type=CommType.irecv, + comm_group=CommGroup.pp_group, + comm_group_size=1, + msg_size=2 + * (args.hidden_size * args.seq_length * args.micro_batch), + stage="forward_step", + additional="recv_next", + ) + ) + # send next + self.workload.append( + LogItem( + comm_type=CommType.isend, + comm_group=CommGroup.pp_group, + comm_group_size=1, + msg_size=2 + * (args.hidden_size * args.seq_length * args.micro_batch), + stage="forward_step", + additional="send_next", + ) + ) + + self.workload.extend(self.model.backward()) + + if pp_rank != 0: + if last_iter: + # send prev + self.workload.append( + LogItem( + comm_type=CommType.isend, + comm_group=CommGroup.pp_group, + comm_group_size=1, + msg_size=2 + * (args.hidden_size * args.seq_length * args.micro_batch), + stage="backward_step", + additional="send_prev", + ) + ) + else: + # send prev recv prev + self.workload.append( + LogItem( + comm_type=CommType.isend, + comm_group=CommGroup.pp_group, + comm_group_size=1, + msg_size=2 + * (args.hidden_size * args.seq_length * args.micro_batch), + stage="backward_step", + additional="send_prev", + ) + ) + self.workload.append( + LogItem( + comm_type=CommType.irecv, + comm_group=CommGroup.pp_group, + comm_group_size=1, + msg_size=2 + * (args.hidden_size * args.seq_length * args.micro_batch), + stage="backward_step", + additional="recv_prev", + ) + ) + + for _ in range(pp_num_warmup_microbatches): + # recv next + if pp_rank != args.pipeline_model_parallel - 1: + self.workload.append( + LogItem( + comm_type=CommType.irecv, + comm_group=CommGroup.pp_group, + comm_group_size=1, + msg_size=2 + * (args.hidden_size * args.seq_length * args.micro_batch), + stage="backward_step", + additional="recv_next", + ) + ) + + self.workload.extend(self.model.backward()) + + # send prev + if pp_rank != 0: + self.workload.append( + LogItem( + comm_type=CommType.isend, + comm_group=CommGroup.pp_group, + comm_group_size=1, + msg_size=2 + * (args.hidden_size * args.seq_length * args.micro_batch), + stage="backward_step", + additional="send_prev", + ) + ) + + def forward(self): + args = self.args + if self.tp_is_enable: + self.workload.append( + LogItem( + comm_type=CommType.broadcast, + comm_group=CommGroup.tp_group, + comm_group_size=self.args.tensor_model_parallel_size, + msg_size=5 * 8, + stage="forward_step", + src=0, + ) + ) + self.workload.append( + LogItem( + comm_type=CommType.broadcast, + comm_group=CommGroup.tp_group, + comm_group_size=self.args.tensor_model_parallel_size, + msg_size=8 * (args.world_size + args.seq_length * args.micro_batch), + stage="forward_step", + src=0, + ) + ) + self.workload.extend(self.model.forward()) + for _ in range(3): + # for bf16, we need to use float32 in loss communication + self.workload.append( + LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.tp_group, + comm_group_size=self.args.tensor_model_parallel_size, + msg_size=args.micro_batch * args.seq_length * 4, + stage="forward_step._VocabParallelCrossEntropy", + ) + ) + # average_losses_across_data_parallel_group + self.workload.append( + LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.dp_group, + comm_group_size=self.args.dp_num, + msg_size=1 * 4, + stage="forward_step.average_losses_across_data_parallel_group", + ) + ) + + def backward(self): + self.workload.extend(self.model.backward()) + + def step(self): + args = self.args + + if args.use_distributed_optimizer: + self.workload.append( + LogItem( + comm_type=CommType.reduce_scatter, + comm_group=CommGroup.dp_group, + comm_group_size=self.args.dp_num, + msg_size=4 * self._get_total_params() // (args.pipeline_model_parallel), + stage="step", + ) + ) + self.workload.append( + LogItem( + comm_type=CommType.all_gather, + comm_group=CommGroup.dp_group, + comm_group_size=self.args.dp_num, + msg_size=2 * self._get_total_params() // (args.pipeline_model_parallel), + stage="step", + ) + ) + else: + # 注意，如果使用过了bf16，那么梯度会使用tf32 + self.workload.append( + LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.dp_group, + comm_group_size=self.args.dp_num, + msg_size=4 * self._get_total_params() // (args.pipeline_model_parallel), + stage="step.finish_grad_sync", + ) + ) + + self.workload.append( + LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.tp_group, + comm_group_size=self.args.tensor_model_parallel_size, + msg_size=2 * self._get_layernorm_params() // (args.pipeline_model_parallel), + stage="step._allreduce_layernorm_grads", + ) + ) + + self.workload.append( + LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.tp_group, + comm_group_size=self.args.tensor_model_parallel_size, + msg_size=4, + stage="step.check_for_nan", + ) + ) + + +if __name__ == "__main__": + args = get_params() + if args.frame == "DeepSeek": + model = DeepSeekV3Model(args) + elif args.frame == "Qwen3": + model = Qwen3Model(args) + elif args.frame == "Qwen3.5": + model = Qwen3_5Model(args) + elif args.frame == "Megatron": + model = MegatronModel(args) + workload_generator = MegatronWorkload(args, model) + workload = workload_generator() + filename = f"{workload_generator.name}_{args.model_name}_sp_{args.enable_sequence_parallel}_iteration_{args.epoch_num}_computationEnable_{args.computation_enable}_{args.world_size}n.csv" + workload.dump(filename) + params = model.parameters() + args.model_param = sum(p.numel() for p in params) + args.activation_memory = 0 + for sub_module in model.child_modules(): + if hasattr(sub_module, "activation_memory"): + args.activation_memory += sub_module.activation_memory() + print(f"model_param: {num_parameters_to_bytes(args, args.model_param)}") + print(f"activation_memory: {num_parameters_to_bytes(args, args.activation_memory)}") + if args.enable_visual: + try: + from visualize.generate import visualize_output + base_name = filename.split(".")[0] + visualize_output(f"./results/mocked_workload/{base_name}_workload.csv",True) + except ImportError: + print("visualize_output is not available because required library is not found") + # WorkloadWriter.write_workload(workload, args, filename) diff --git a/aicb/workload_generator/mocked_model/MockedModel.py b/aicb/workload_generator/mocked_model/MockedModel.py new file mode 100644 index 000000000..4a9080829 --- /dev/null +++ b/aicb/workload_generator/mocked_model/MockedModel.py @@ -0,0 +1,168 @@ +""" +Copyright (c) 2021, Alibaba Group; +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import math +from typing import List, Tuple +from enum import Enum + +class InferencePhase(Enum): + PREFILL = "prefill" + DECODE = "decode" + +class MockedParam: + def __init__(self, shape: Tuple, elem_size=2, name=None) -> None: + self.shape = shape + self._numel = math.prod(shape) + self._elem_size = elem_size + self.name = name if name is not None else "Unknown" + + def numel(self): + return self._numel + + def elem_size(self): + return self._elem_size + + def msg_size(self): + return self._numel * self._elem_size + + def get_shape(self): + return self.shape + + # def name(self): + # return self.param_name + + +def _unpack_params(value: object) -> List[MockedParam]: + if isinstance(value, MockedParam): + return [value] + elif isinstance(value, MockedModel): + return value.parameters() + elif isinstance(value, dict): + params = [] + for k, v in value.items(): + params += _unpack_params(v) + return params + elif isinstance(value, (list, tuple)): + params = [] + for v in value: + params += _unpack_params(v) + return params + else: + return [] + + +def _child_modules(value: object) -> List["MockedModel"]: + if isinstance(value, MockedModel): + modules = [value] + modules.extend(_child_modules(value.__dict__)) + return modules + elif isinstance(value, dict): + modules = [] + for k, v in value.items(): + modules += _child_modules(v) + return modules + elif isinstance(value, (list, tuple)): + modules = [] + for v in value: + modules += _child_modules(v) + return modules + else: + return [] + + +class MockedParamsBase: + def __init__(self, model_name: str, frame: str, config_file=None, args=None): + self.model_name = model_name + self.frame = frame + + # Load from config file if provided + if config_file: + self.load_from_config(config_file) + + # Override with command line args if provided + if args: + self.load_from_args(args) + + if self.world_size < self.tensor_model_parallel_size or self.world_size < self.expert_model_parallel_size or self.world_size < 1: + raise ValueError(f"Invalid world size: world_size={self.world_size}, tensor_model_parallel_size={self.tensor_model_parallel_size}, expert_model_parallel_size={self.expert_model_parallel_size}") + + def load_from_config(self, config_file): + import json + try: + with open(config_file, 'r') as f: + config_data = json.load(f) + + # Update attributes with values from config file + for key, value in config_data.items(): + setattr(self, key, value) + + except FileNotFoundError: + #error, quit + print(f"[ERRPR] Config file '{config_file}' not found.") + exit(1) + except json.JSONDecodeError: + print(f"[ERRPR] File '{config_file}' is not valid JSON.") + exit(1) + + def load_from_args(self, args): + """Load parameters from command line arguments""" + # List of parameters that can be overridden by command line args + arg_params = [ + 'aiob_enable', + 'seq_length', + 'micro_batch', + 'world_size', + 'tensor_model_parallel_size', + 'expert_model_parallel_size', + 'pipeline_model_parallel', + 'moe_enable', + 'result_dir', + 'phase', + 'aiob_forward_loops' + ] + + # Override parameters with values from args if they exist + for param in arg_params: + if hasattr(args, param): + setattr(self, param, getattr(args, param)) + + +class MockedModel: + def __init__(self) -> None: + self._pre_forward_hook = [] + self._post_forward_hook = [] + self._pre_backward_hook = [] + self._post_backward_hook = [] + + def parameters(self) -> List[MockedParam]: + return _unpack_params(self.__dict__) + + def child_modules(self) -> List["MockedModel"]: + return _child_modules(self.__dict__) + + def register_forward_pre_hook(self, fn): + self._pre_forward_hook.append(fn) + + def register_backward_pre_hook(self, fn): + self._pre_backward_hook.append(fn) + + def register_forward_post_hook(self, fn): + self._post_forward_hook.append(fn) + + def register_backward_post_hook(self, fn): + self._post_backward_hook.append(fn) + + +class Linear(MockedModel): # alias for LlamaRMSNorm, Embedding, LlamaRotaryEmbedding + def __init__(self, in_feature, out_feature): + self.weight = MockedParam((in_feature, out_feature)) diff --git a/aicb/workload_generator/mocked_model/__init__.py b/aicb/workload_generator/mocked_model/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/aicb/workload_generator/mocked_model/inference/AiobDeepSeek.py b/aicb/workload_generator/mocked_model/inference/AiobDeepSeek.py new file mode 100644 index 000000000..dfed7ee6e --- /dev/null +++ b/aicb/workload_generator/mocked_model/inference/AiobDeepSeek.py @@ -0,0 +1,496 @@ +import torch +import workload_generator.mocked_model.inference.MockedDeepSeek as MockedDeepSeek +from workload_generator.mocked_model.MockedModel import InferencePhase +from utils.utils import * +from utils.deepgemm_utils import * +import torch.nn.functional as F +import random +from typing import Tuple +import triton +import numpy as np +from flash_mla import flash_mla_with_kvcache, get_mla_metadata, flash_mla_sparse_fwd + +def to_float8(x, dtype=torch.float8_e4m3fn): + finfo = torch.finfo(dtype) + min_val, max_val = x.aminmax() + amax = torch.maximum(min_val.abs(), max_val.abs()).clamp(min=1e-12) + scale = finfo.max / amax + x_scl_sat = (x * scale).clamp(min=finfo.min, max=finfo.max) + return x_scl_sat.to(dtype), scale.float().reciprocal() +def construct_bmm(b: int, m: int, k: int, n: int) -> \ + Tuple[torch.Tensor, torch.Tensor, Tuple[torch.Tensor, torch.Tensor], + Tuple[torch.Tensor, torch.Tensor], torch.Tensor, torch.Tensor]: + x = torch.randn((b, m, k), device='cuda:0', dtype=torch.bfloat16) + y = torch.randn((b, n, k), device='cuda:0', dtype=torch.bfloat16) + out = torch.empty((b, m, n), device='cuda:0', dtype=torch.bfloat16) + + y = y.transpose(-2, -1) + ref_out = torch.bmm(x, y) + + x_fp8 = to_float8(x) + y_fp8 = to_float8(y) + return x, y, x_fp8, y_fp8, out, ref_out + +class DeepSeekEmbedding(torch.nn.Module): + def __init__(self, args=None): + super(DeepSeekEmbedding, self).__init__() + self.tp = args.tensor_model_parallel_size + hidden_size = args.hidden_size + # max_position_embeddings = args.max_position_embeddings + self.vocab_size = args.vocab_size + device = torch.device("cuda:0") + torch.cuda.set_device(device) + self.device = device + self.dtype = torch.bfloat16 + self.weight = torch.randint( + low=0, + high=1, # 设置权重范围内的随机值，假设权重值在 0 到 1之间 + size=(self.vocab_size, hidden_size), + device=device, + dtype=torch.bfloat16 + ) + + @cuda_timing_decorator + def _apply(self, input): + if self.tp > 1: + # Build the mask. + input_mask = (input < 0) | (input >= math.ceil(self.vocab_size / self.tp)) + # Mask the input. + masked_input = input.clone() - 0 + masked_input[input_mask] = 0 + else: + masked_input = input + + embeddings = F.embedding(masked_input,self.weight) + return embeddings + def forward(self,input): + input = input.to(self.device) + result, emb_time = self._apply(input) + + result = result.to(self.dtype) + + return result, emb_time + +class DeepSeekRMSNorm(torch.nn.Module): + def __init__(self, args=None): + super(DeepSeekRMSNorm, self).__init__() + self.hidden_size = args.hidden_size + + self.tp = args.tensor_model_parallel_size + device = torch.device("cuda:0") + torch.cuda.set_device(device) + self.dtype = torch.bfloat16 + self.norm_weight = torch.rand(self.hidden_size, device=device).to(self.dtype) + + @cuda_timing_decorator + def _apply(self, hidden_states): + output_norm = F.rms_norm(hidden_states, (self.hidden_size,), self.norm_weight, 1e-6) + return output_norm + + def forward(self, hidden_states): + norm_output, norm_time = self._apply(hidden_states) + + return norm_output, norm_time + + + +class DeepSeekAtten(torch.nn.Module): + def __init__(self, args=None): + super(DeepSeekAtten, self).__init__() + self.tp = args.tensor_model_parallel_size + self.hidden_size = args.hidden_size + self.num_heads = args.head_num + self.batch_size = args.micro_batch + self.d_kv_c = args.d_kv_c + self.d_q_c = args.d_q_c + self.d_r = args.d_r + self.d_q = args.d_q + self.d_kv = args.d_kv + self.args = args + + def _qkv_compression(self, m): + k = self.hidden_size #7168 + n = self.d_kv_c + self.d_q_c + self.d_r #2112 + print(f'm={m}, k={k}, n={n}') + x_fp8, y_fp8, c, out, ref_out = construct(m, k, n) + # deep_gemm.gemm_fp8_fp8_bf16_nt(x_fp8, y_fp8, out) + deep_gemm.fp8_gemm_nt(x_fp8, y_fp8, out, c=c, disable_ue8m0_cast=True, recipe = None) + diff = calc_diff(out, ref_out) + assert diff < 0.001, f'{m=}, {k=}, {n=}, {diff:.5f}' + x_fp8, y_fp8, c, out, ref_out = construct(m, k, n) + def test_func(): + # deep_gemm.gemm_fp8_fp8_bf16_nt(x_fp8, y_fp8, out) + deep_gemm.fp8_gemm_nt(x_fp8, y_fp8, out, c=c, disable_ue8m0_cast=True, recipe = None) + t = bench_kineto(test_func, 'fp8_gemm', suppress_kineto_output=True) + return t + + def _q_uncrompression(self, m): + k = self.d_q_c #1536 + n = self.num_heads *(self.d_q + self.d_r) // self.tp + print(f'm={m}, k={k}, n={n}') + x_fp8, y_fp8, c, out, ref_out = construct(m, k, n) + # deep_gemm.gemm_fp8_fp8_bf16_nt(x_fp8, y_fp8, out) + deep_gemm.fp8_gemm_nt(x_fp8, y_fp8, out, c=c, disable_ue8m0_cast=True, recipe = None) + diff = calc_diff(out, ref_out) + assert diff < 0.001, f'{m=}, {k=}, {n=}, {diff:.5f}' + x_fp8, y_fp8, c, out, ref_out = construct(m, k, n) + def test_func(): + # deep_gemm.gemm_fp8_fp8_bf16_nt(x_fp8, y_fp8, out) + deep_gemm.fp8_gemm_nt(x_fp8, y_fp8, out, c=c, disable_ue8m0_cast=True, recipe = None) + t = bench_kineto(test_func, 'fp8_gemm', suppress_kineto_output=True) + return t + def _q_bmm(self, m): + b = self.num_heads //self.tp + k = self.d_q + n = self.d_kv_c #512 + print(f'm={m}, b={b}, k={k}, n={n}') + x_bf16, y_bf16, x_fp8, y_fp8, out, ref_out = construct_bmm( + b, m, k, n) + def test_func(): + x_bf16, y_bf16, x_fp8, y_fp8, out, ref_out = construct_bmm( + b, m, k, n) + # with nvtx.range("matmul"): + out = torch.bmm(x_bf16, y_bf16) + + try: + t = bench_kineto(test_func, "gemm_bf16", suppress_kineto_output=True) + except: + try: + t = bench_kineto(test_func, "gemm_relu_bf16", suppress_kineto_output=True) + except: + # H20 use a strange kernel named "nvjet_tst_xxx_v_bz_TNN" to perform + t = bench_kineto(test_func, "nvjet_tst", suppress_kineto_output=True) + return t + + def _attention(self): + device = torch.device("cuda:0") + torch.set_default_dtype(torch.bfloat16) + torch.set_default_device(device) + torch.cuda.set_device(device) + torch.manual_seed(0) + random.seed(0) + + phase = getattr(self.args, "phase", InferencePhase.DECODE.value) + if phase == InferencePhase.DECODE.value: + b = self.batch_size + s_q = 1 + mean_sk = self.args.seq_length + 1 # TODO not a good solution. + elif phase == InferencePhase.PREFILL.value: + b = 1 + s_q = self.args.seq_length + mean_sk = self.args.seq_length + h_q = self.num_heads //self.tp + d = self.d_kv_c + self.d_r + q = torch.randn(b, s_q, h_q, d) + h_kv = 1 + + if phase == InferencePhase.DECODE.value: + #From DeepSeek-simulator + cache_seqlens = torch.full((b,), mean_sk, dtype=torch.int32) + for i in range(b): + cache_seqlens[i] = max( + random.normalvariate(mean_sk, mean_sk / 2), s_q) + # total_seqlens = cache_seqlens.sum().item() + # mean_seqlens = cache_seqlens.float().mean().int().item() + max_seqlen = cache_seqlens.max().item() + max_seqlen_pad = triton.cdiv(max_seqlen, 256) * 256 + block_size = 64 + + block_table = torch.arange( + b * max_seqlen_pad // block_size, dtype=torch.int32 + ).view(b, max_seqlen_pad // block_size) + blocked_k = torch.randn(block_table.numel(), block_size, h_kv, d) + for i in range(b): + blocked_k.view(b, max_seqlen_pad, h_kv, d)[i, cache_seqlens[i].item():] = ( + float("nan") + ) + # blocked_v = blocked_k[..., :self.d_kv_c] + + tile_scheduler_metadata, num_splits = get_mla_metadata( + cache_seqlens, s_q * h_q // h_kv, h_kv + ) + + def flash_mla(): + return flash_mla_with_kvcache( + q, + blocked_k, + block_table, + cache_seqlens, + self.d_kv_c, + tile_scheduler_metadata, + num_splits, + causal=True, + ) + t = triton.testing.do_bench(flash_mla) + return t + elif phase == InferencePhase.PREFILL.value: + # from https://github.com/deepseek-ai/FlashMLA/blob/main/tests/test_flash_mla_prefill.py + s_kv = max(s_q,4096) #TODO + topk = 2048 #TODO + + kv = torch.randn(b, s_kv, h_kv, d) + indices = torch.full((b, s_q, h_kv, topk), s_kv, dtype=torch.int32) + sm_scale = 1 / math.sqrt(d) + def flash_mla(): + return flash_mla_sparse_fwd(q.squeeze(0), kv.squeeze(0), indices.squeeze(0), sm_scale) + t = triton.testing.do_bench(flash_mla) + return t + + def _O_bmm(self, m): + b = self.num_heads //self.tp + k = self.d_kv_c #512 + n = self.d_kv #128 + print(f'm={m}, b={b}, k={k}, n={n}') + x_bf16, y_bf16, x_fp8, y_fp8, out, ref_out = construct_bmm( + b, m, k, n) + def test_func(): + x_bf16, y_bf16, x_fp8, y_fp8, out, ref_out = construct_bmm( + b, m, k, n) + # with nvtx.range("matmul"): + out = torch.bmm(x_bf16, y_bf16) + + try: + t = bench_kineto(test_func, "gemm_bf16", suppress_kineto_output=True) + except: + try: + t = bench_kineto(test_func, "gemm_relu_bf16", suppress_kineto_output=True) + except: + # H20 uses a strange kernel named "nvjet_tst_xxx_v_bz_TNN" + t = bench_kineto(test_func, "nvjet_tst", suppress_kineto_output=True) + return t + + def _O_proj(self, m): + k = self.num_heads * self.d_kv //self.tp + n = self.hidden_size + print(f'm={m}, k={k}, n={n}') + x_fp8, y_fp8, c, out, ref_out = construct(m, k, n) + # deep_gemm.gemm_fp8_fp8_bf16_nt(x_fp8, y_fp8, out) + deep_gemm.fp8_gemm_nt(x_fp8, y_fp8, out, c=c, disable_ue8m0_cast=True, recipe = None) + diff = calc_diff(out, ref_out) + assert diff < 0.001, f'{m=}, {k=}, {n=}, {diff:.5f}' + x_fp8, y_fp8, c, out, ref_out = construct(m, k, n) + def test_func(): + # deep_gemm.gemm_fp8_fp8_bf16_nt(x_fp8, y_fp8, out) + deep_gemm.fp8_gemm_nt(x_fp8, y_fp8, out, c=c, disable_ue8m0_cast=True, recipe = None) + t = bench_kineto(test_func, 'fp8_gemm', suppress_kineto_output=True) + return t + + def forward(self): + phase = getattr(self.args, "phase", InferencePhase.DECODE.value) + if phase == InferencePhase.DECODE.value: + m = self.batch_size + elif phase == InferencePhase.PREFILL.value: + m = self.args.seq_length + qkv_compression_time = self._qkv_compression(m) + print("matrix 1 is:" , qkv_compression_time*1e6) + q_uncrompression_time = self._q_uncrompression(m) + print("matrix 2 is:" , q_uncrompression_time*1e6) + q_bemm = self._q_bmm(m) + print("matrix 3 is:" , q_bemm*1e6) + atten_qkv = qkv_compression_time + q_uncrompression_time + q_bemm + atten_core = self._attention() + linear_bmm = self._O_bmm(m) + print("matrix 9 is:" , linear_bmm*1e6) + linear_proj = self._O_proj(m) + print("matrix 4 is:" , linear_proj*1e6) + atten_linear = linear_bmm + linear_proj + + return atten_qkv, atten_core, atten_linear + + + +class DeepSeekMLP(torch.nn.Module): + def __init__(self, args=None): + super(DeepSeekMLP, self).__init__() + # self.tp = 1 + self.batch_size = args.micro_batch + self.hidden_size = args.hidden_size + self.expert_dim = args.expert_dim + self.seq_length = args.seq_length + self.args = args + + def _up_gate(self, m): + k = self.hidden_size #7168 + n = self.expert_dim * 2 #不开TP + print(f'm={m}, k={k}, n={n}') + x_fp8, y_fp8, c, out, ref_out = construct(m, k, n) + # deep_gemm.gemm_fp8_fp8_bf16_nt(x_fp8, y_fp8, out) + deep_gemm.fp8_gemm_nt(x_fp8, y_fp8, out, c=c, disable_ue8m0_cast=True, recipe = None) + diff = calc_diff(out, ref_out) + assert diff < 0.001, f'{m=}, {k=}, {n=}, {diff:.5f}' + x_fp8, y_fp8, c, out, ref_out = construct(m, k, n) + def test_func(): + # deep_gemm.gemm_fp8_fp8_bf16_nt(x_fp8, y_fp8, out) + deep_gemm.fp8_gemm_nt(x_fp8, y_fp8, out, c=c, disable_ue8m0_cast=True, recipe = None) + t = bench_kineto(test_func, 'fp8_gemm', suppress_kineto_output=True) + return t + + def _down(self, m): + k = self.expert_dim #不开TP + n = self.hidden_size + print(f'm={m}, k={k}, n={n}') + x_fp8, y_fp8, c, out, ref_out = construct(m, k, n) + # deep_gemm.gemm_fp8_fp8_bf16_nt(x_fp8, y_fp8, out) + deep_gemm.fp8_gemm_nt(x_fp8, y_fp8, out, c=c, disable_ue8m0_cast=True, recipe = None) + diff = calc_diff(out, ref_out) + assert diff < 0.001, f'{m=}, {k=}, {n=}, {diff:.5f}' + x_fp8, y_fp8, c, out, ref_out = construct(m, k, n) + def test_func(): + # deep_gemm.gemm_fp8_fp8_bf16_nt(x_fp8, y_fp8, out) + deep_gemm.fp8_gemm_nt(x_fp8, y_fp8, out, c=c, disable_ue8m0_cast=True, recipe = None) + t = bench_kineto(test_func, 'fp8_gemm', suppress_kineto_output=True) + return t + + def forward(self): + phase = getattr(self.args, "phase", InferencePhase.DECODE.value) + if phase == InferencePhase.DECODE.value: + m = self.batch_size + elif phase == InferencePhase.PREFILL.value: + m = self.seq_length + up_gate = self._up_gate(m) + print("MLP:UP: ", up_gate*1e6) + down = self._down(m) + print("MLP:DOWN: ", down*1e6) + return up_gate , down + +class DeepSeekMOE(torch.nn.Module): + def __init__(self, args=None): + super(DeepSeekMOE, self).__init__() + self.batch_size = args.micro_batch + self.hidden_size = args.hidden_size + self.expert_dim = args.expert_dim + self.num_experts = args.router_expert + self.tp = args.tensor_model_parallel_size + self.dp = args.world_size // self.tp // args.pipeline_model_parallel + self.ep = self.tp * self.dp + self.topk = args.moe_router_topk + self.seq_length = args.seq_length + self.args = args + + def _up_gate(self, m): + num_groups = self.num_experts // self.ep + strategy = getattr(self.args, "moe_routing_strategy", Strategy.RoundRobin) + expected_m_per_group = get_ep_expected_m_per_group( + m, num_groups, self.topk, self.ep, strategy + ) + + n = self.expert_dim * 2 + k = self.hidden_size + + phase = getattr(self.args, "phase", InferencePhase.DECODE.value) + + if phase == InferencePhase.DECODE.value: + t = bench_masked(num_groups, expected_m_per_group, k, n) + elif phase == InferencePhase.PREFILL.value: + t = bench_contiguous(num_groups, expected_m_per_group, k, n) + + return t + + def _down(self, m): + num_groups = self.num_experts // self.ep + strategy = getattr(self.args, "moe_routing_strategy", Strategy.RoundRobin) + expected_m_per_group = get_ep_expected_m_per_group( + m, num_groups, self.topk, self.ep, strategy + ) + + n = self.hidden_size + k = self.expert_dim + + phase = getattr(self.args, "phase", InferencePhase.DECODE.value) + + if phase == InferencePhase.DECODE.value: + t = bench_masked(num_groups, expected_m_per_group, k, n) + elif phase == InferencePhase.PREFILL.value: + t = bench_contiguous(num_groups, expected_m_per_group, k, n) + + return t + + def forward(self): + phase = getattr(self.args, "phase", InferencePhase.DECODE.value) + if phase == InferencePhase.DECODE.value: + m = self.batch_size + elif phase == InferencePhase.PREFILL.value: + m = self.args.seq_length + up_gate = self._up_gate(m) + down = self._down(m) + + return up_gate, down + + +class DeepSeekModel(torch.nn.Module): + def __init__(self, args=None): + super(DeepSeekModel, self).__init__() + self.time_list = {} + self.args = args + self.Embedding = DeepSeekEmbedding(self.args) + # self.transformer = DeepSeekTransformer(self.args) + self.RMSNorm = DeepSeekRMSNorm(self.args) + self.Attention = DeepSeekAtten(self.args) + self.MLP = DeepSeekMLP(self.args) + self.MOE = DeepSeekMOE(self.args) + self.aiob_forward_loops = getattr(args, "aiob_forward_loops", 10) + + + def forward(self): + # Emb_output, Emb_time = self.Embedding(input) + # print("Embedding time: ", Emb_time) + # self.time_list.setdefault("Emb", []).append({"time_gpu": Emb_time}) + for i in range(self.aiob_forward_loops): + # norm_output, norm_time1 = self.RMSNorm(Emb_output) + # self.time_list.setdefault("RMSNorm1", []).append( + # {"time_gpu": norm_time1} + # ) + atten_qkv, atten_core, atten_linear = self.Attention() + self.time_list.setdefault("atten_qkv", []).append( + {"time_gpu": atten_qkv* 1e6} + ) + self.time_list.setdefault("atten_flash", []).append( + {"time_gpu": atten_core* 1e3} + ) + self.time_list.setdefault("atten_linear", []).append( + {"time_gpu": atten_linear* 1e6} + ) + # print("norm_time time: ", norm_time1) + print(f'qkv_core time: , {atten_qkv * 1e6}') + print(f'qkv_attention time: , {atten_core* 1e3}') + print(f'atten_linear time: , {atten_linear * 1e6}') + # self.time_list.setdefault("RMSNorm2", []).append( + # {"time_gpu": norm_time2} + # ) + # print("norm_time time: ", norm_time2) + mlp_up, mlp_down = self.MLP() + self.time_list.setdefault("mlp_up", []).append( + {"time_gpu": mlp_up* 1e6} + ) + self.time_list.setdefault("mlp_down", []).append( + {"time_gpu": mlp_down* 1e6} + ) + print(f'mlp_time time: , {mlp_up * 1e6}') + print(f'mlp_down time: , {mlp_down * 1e6}') + up_gate_moe, down_moe = self.MOE() + self.time_list.setdefault("moe_up_gate", []).append( + {"time_gpu": up_gate_moe* 1e6} + ) + self.time_list.setdefault("moe_down", []).append( + {"time_gpu": down_moe* 1e6} + ) + print(f'up_gate_moe time: , {up_gate_moe * 1e6}') + print(f'down_moe time: , {down_moe * 1e6}') + print(self.time_list) + result_dir = "./results/aiob_outputs" + if not os.path.isdir(result_dir): + os.makedirs(result_dir) + filename = f"{self.args.model_name}_time_list_stats.txt" + file_name = os.path.join(result_dir, filename) + calculate_stats(self.time_list, file_name) + filepath = write_time(self.time_list, self.args) + process_all_keys(filepath) + return filepath + + +if __name__ == "__main__": + args = MockedDeepSeek.DeepSeekParams() + x = torch.randint(0, args.vocab_size, (2, 1)) + model = DeepSeekModel(args) + filepath = model(x) + print(filepath) \ No newline at end of file diff --git a/aicb/workload_generator/mocked_model/inference/AiobQwen3Moe.py b/aicb/workload_generator/mocked_model/inference/AiobQwen3Moe.py new file mode 100644 index 000000000..77ba3009a --- /dev/null +++ b/aicb/workload_generator/mocked_model/inference/AiobQwen3Moe.py @@ -0,0 +1,428 @@ +import torch +import workload_generator.mocked_model.inference.MockedQwen3Moe as MockedQwen3Moe +from workload_generator.mocked_model.MockedModel import InferencePhase +from utils.utils import * +from utils.deepgemm_utils import * +import torch.nn.functional as F +import random +from typing import Optional, Union, Tuple +# import triton +import numpy as np +from vllm.model_executor.layers.quantization.utils.fp8_utils import per_token_group_quant_fp8 +from vllm import _custom_ops as vllm_ops +from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding, get_rope +from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk +import flashinfer + +# from benchmark_rmsnorm.py +def rmsnorm_vllm( + x: torch.Tensor, + weight: torch.Tensor, + residual: Optional[torch.Tensor] = None, + eps: float = 1e-6, +): + orig_shape = x.shape + x = x.view(-1, x.shape[-1]) + if residual is not None: + residual = residual.view(-1, residual.shape[-1]) + + if residual is not None: + vllm_ops.fused_add_rms_norm(x, residual, weight, eps) + output = (x, residual) + else: + out = torch.empty_like(x) + vllm_ops.rms_norm(out, x, weight, eps) + output = out + + if isinstance(output, tuple): + output = (output[0].view(orig_shape), output[1].view(orig_shape)) + else: + output = output.view(orig_shape) + return output + +class Qwen3MoeAttention(torch.nn.Module): + def __init__(self, args=None): + super(Qwen3MoeAttention, self).__init__() + self.args = args + self.tp = self.args.tensor_model_parallel_size + + def _norm(self, m): + head_dim = self.args.head_dim + x = torch.randn(m,head_dim, device='cuda', dtype=torch.bfloat16) + weight = torch.ones(head_dim, dtype=torch.bfloat16, device='cuda') + residual = torch.randn(m,head_dim, device='cuda', dtype=torch.bfloat16) + def test_func(): + rmsnorm_vllm(x.clone(), weight, residual.clone(), self.args.rms_norm_eps) + t = bench_kineto(test_func, 'vllm::fused_ad', suppress_kineto_output=True) + return t + def _attn_quant(self, qk_or_o, m): + if qk_or_o: + n = self.args.hidden_size + else: + n = self.args.num_attention_heads * self.args.head_dim // self.tp + input = torch.randn(m, n, device='cuda', dtype=torch.bfloat16) + def test_func(): + per_token_group_quant_fp8(input, 128, column_major_scales=True) + + candidates = [ + 'per_token_group_quant_fp8', # old version + 'per_token_group_quant_8bit', # new version + ] + last_err = None + for names in candidates: + try: + t = bench_kineto(test_func, names, suppress_kineto_output=True) + return t + except Exception as e: + last_err = e + continue + + raise RuntimeError(f"bench_kineto failed for all candidates: {last_err}") + + def _attn_proj(self, qk_or_o, m): + if qk_or_o: + n = (self.args.num_attention_heads + self.args.num_key_value_heads * 2) * self.args.head_dim // self.tp + k = self.args.hidden_size + else: + n = self.args.hidden_size + k = self.args.num_attention_heads * self.args.head_dim // self.tp + x_fp8, y_fp8, c, out, ref_out = construct(m, k, n) + # deep_gemm.gemm_fp8_fp8_bf16_nt(x_fp8, y_fp8, out) + deep_gemm.fp8_gemm_nt(x_fp8, y_fp8, out, c=c, disable_ue8m0_cast=True, recipe = None) + diff = calc_diff(out, ref_out) + assert diff < 0.001, f'{m=}, {k=}, {n=}, {diff:.5f}' + x_fp8, y_fp8, c, out, ref_out = construct(m, k, n) + def test_func(): + # deep_gemm.gemm_fp8_fp8_bf16_nt(x_fp8, y_fp8, out) + deep_gemm.fp8_gemm_nt(x_fp8, y_fp8, out, c=c, disable_ue8m0_cast=True, recipe = None) + t = bench_kineto(test_func, 'fp8_gemm', suppress_kineto_output=True) + return t + def _qk_norm(self, q_or_k, m): + head_dim = self.args.head_dim + if q_or_k: + n = self.args.num_attention_heads * head_dim // self.tp + else: + n = self.args.num_key_value_heads * head_dim // self.tp + x = torch.randn(m,n, device='cuda', dtype=torch.bfloat16) + x = x.view(*x.shape[:-1], x.shape[-1] // head_dim, head_dim) + weight = torch.ones(head_dim, dtype=torch.bfloat16, device='cuda') + def test_func(): + rmsnorm_vllm(x.clone(), weight, residual=None, eps = self.args.rms_norm_eps) + t = bench_kineto(test_func, 'vllm::rms_norm_kernel', suppress_kineto_output=True) + return t + + def _rotary_emb(self, m): + head_dim = self.args.head_dim + n_q = self.args.num_attention_heads * head_dim // self.tp + n_k = self.args.num_key_value_heads * head_dim // self.tp + + q = torch.randn(m, n_q, device='cuda', dtype=torch.bfloat16) + k = torch.randn(m, n_k, device='cuda', dtype=torch.bfloat16) + pos = torch.arange(0, m, device='cuda') + rotary_emb = get_rope( + head_dim, + rotary_dim=head_dim, + max_position=self.args.max_position_embeddings, + base=self.args.rope_theta, + rope_scaling=self.args.rope_scaling, + dual_chunk_attention_config = getattr(self.args, + "dual_chunk_attention_config", + None), + ) + def test_func(): + rotary_emb(pos, q, k) + + t = bench_kineto(test_func, 'rotary_embedding_kernel', suppress_kineto_output=True) + return t + + def _attn_core(self, kv_lens): + block_size = 32 + num_blocks = 32768 + + head_size = self.args.head_dim + num_query_heads = self.args.num_attention_heads // self.tp + num_kv_heads = self.args.num_key_value_heads// self.tp + dtype = torch.bfloat16 + + + # based on vllm:test_flashinfer_decode_with_paged_kv + torch.set_default_device("cuda") + num_seqs = len(kv_lens) + assert num_query_heads % num_kv_heads == 0 + max_kv_len = max(kv_lens) + scale = head_size**-0.5 + soft_cap = None + + query = torch.randn(num_seqs, num_query_heads, head_size, dtype=dtype) + + key_value_cache = torch.randn(num_blocks, + 2, + block_size, + num_kv_heads, + head_size, + dtype=dtype) + key_cache = key_value_cache[:, 0, :, :, :].squeeze(1) + value_cache = key_value_cache[:, 1, :, :, :].squeeze(1) + + max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size + block_tables = torch.randint(0, + num_blocks, + (num_seqs, max_num_blocks_per_seq), + dtype=torch.int32) + + kv_indptr = [0] + kv_indices = [] + kv_last_page_lens = [] + for i in range(num_seqs): + seq_len = kv_lens[i] + assert seq_len > 0 + num_blocks = (seq_len + block_size - 1) // block_size + kv_indices.extend(block_tables[i, :num_blocks]) + kv_indptr.append(kv_indptr[-1] + num_blocks) + kv_last_page_len = seq_len % block_size + if kv_last_page_len == 0: + kv_last_page_len = block_size + kv_last_page_lens.append(kv_last_page_len) + + kv_indptr = torch.tensor(kv_indptr, dtype=torch.int32) + kv_indices = torch.tensor(kv_indices, dtype=torch.int32) + kv_last_page_lens = torch.tensor(kv_last_page_lens, dtype=torch.int32) + + # 128 MB + workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8) + wrapper = flashinfer.\ + BatchDecodeWithPagedKVCacheWrapper(workspace_buffer, "NHD", + use_tensor_cores=( + (num_query_heads//num_kv_heads) > 4) + ) + wrapper.plan(kv_indptr, + kv_indices, + kv_last_page_lens, + num_query_heads, + num_kv_heads, + head_size, + block_size, + "NONE", + q_data_type=dtype, + kv_data_type=dtype, + logits_soft_cap=soft_cap) + + def test_func(): + wrapper.run(query, key_value_cache) + t = bench_kineto(test_func, 'BatchPrefillWithPagedKVCacheKernel', suppress_kineto_output=True) + return t + + def forward(self): + phase = getattr(self.args, "phase", InferencePhase.DECODE.value) + if phase == InferencePhase.DECODE.value: + m = self.args.micro_batch + kv_lens = [self.args.seq_length] * m + elif phase == InferencePhase.PREFILL.value: + m = self.args.seq_length + kv_lens = [self.args.seq_length] * 1 + + #norm + pre_attention_layernorm = self._norm(m) + + # qkv + qkv_quant = self._attn_quant(True,m) + qkv_proj = self._attn_proj(True,m) + q_norm = self._qk_norm(True,m) + k_norm = self._qk_norm(False,m) + attn_qkv = qkv_quant + qkv_proj + q_norm + k_norm + + # rotary_embedding + rotary_emb = self._rotary_emb(m) + + # core + attn = self._attn_core(kv_lens) + + # o + o_quant = self._attn_quant(False,m) + o_proj = self._attn_proj(False,m) + attn_o = o_quant + o_proj + + #norm + post_attention_layernorm = self._norm(m) + + + return pre_attention_layernorm, attn_qkv, rotary_emb, attn, attn_o, post_attention_layernorm + +class Qwen3MoeSparseMoeBlock(torch.nn.Module): + def __init__(self, args): + super(Qwen3MoeSparseMoeBlock, self).__init__() + self.args = args + self.hidden_size = args.hidden_size + self.num_experts = args.num_experts + self.tp = args.tensor_model_parallel_size + self.dp = args.world_size // self.tp // args.pipeline_model_parallel + self.ep = self.tp * self.dp + self.topk = args.num_experts_per_tok + + def _route_gate(self, m): + # ReplicatedLinear + x = torch.randn(m, self.hidden_size, device='cuda', dtype=torch.bfloat16) + w = torch.randn(self.num_experts, self.hidden_size, device='cuda', dtype=torch.bfloat16) + + def test_func(): + F.linear(x, w) + + candidates = [ + ("cutlass", "cublasLt::splitK"), + ("nvjet_tst", "cublasLt::splitK"), + "cutlass", + "nvjet_tst", + ] + last_err = None + for names in candidates: + try: + res = bench_kineto(test_func, names, suppress_kineto_output=True) + return sum(res) if isinstance(res, (list, tuple)) else float(res) + except Exception as e: + last_err = e + continue + + raise RuntimeError(f"bench_kineto failed for all candidates: {last_err}") + + def _select_experts(self, total_tokens): + # FusedMoE.select_experts + hidden_states = torch.randn(total_tokens, self.hidden_size, device='cuda', dtype=torch.bfloat16) + router_logits = torch.randn(total_tokens, self.num_experts, device='cuda', dtype=torch.bfloat16) + + def test_func(): + fused_topk(hidden_states=hidden_states, + gating_output=router_logits, + topk=self.topk, + renormalize=True) + t = bench_kineto(test_func, 'moe::topkGatingSoftmax', suppress_kineto_output=True) + return t + + def _moe_gate_proj(self, up_or_down, m): + num_groups = self.num_experts // self.ep + strategy = getattr(self.args, "moe_routing_strategy", Strategy.RoundRobin) + expected_m_per_group = get_ep_expected_m_per_group( + m, num_groups, self.topk, self.ep, strategy + ) + if up_or_down: + n = 2 * self.args.moe_intermediate_size + k = self.hidden_size + else: + n = self.hidden_size + k = self.args.moe_intermediate_size + + phase = getattr(self.args, "phase", InferencePhase.DECODE.value) + + if phase == InferencePhase.DECODE.value: + t = bench_masked(num_groups, expected_m_per_group, k, n) + elif phase == InferencePhase.PREFILL.value: + t = bench_contiguous(num_groups, expected_m_per_group, k, n) + + return t + + def _moe_act(self, m): + # SiluAndMul + num_groups=self.num_experts // self.ep + total = m * self.dp * self.topk // self.num_experts + n = 2 * self.args.moe_intermediate_size + + total_max = total # TODO: maybe should set in config + intermediate_cache1 = torch.randn((num_groups * total_max, n), + device='cuda', + dtype=torch.bfloat16) + intermediate_cache2 = torch.randn((num_groups * total_max, n // 2), + device='cuda', + dtype=torch.bfloat16) + + def test_func(): + torch.ops._C.silu_and_mul( + intermediate_cache2.unflatten(0, (num_groups, total_max)), + intermediate_cache1.unflatten(0, (num_groups, total_max))) + # vllm::silu vllm::act_and_mul + t = bench_kineto(test_func, "vllm::silu", suppress_kineto_output=True) + return t + + + def forward(self): + phase = getattr(self.args, "phase", InferencePhase.DECODE.value) + if phase == InferencePhase.DECODE.value: + m = self.args.micro_batch + total_tokens = m # TODO: maybe should set output seq_length + elif phase == InferencePhase.PREFILL.value: + m = self.args.seq_length + total_tokens = m + + route_gate = self._route_gate(m) + route_select_experts = self._select_experts(total_tokens) + moe_up = self._moe_gate_proj(True, m) + moe_act = self._moe_act(m) + moe_down = self._moe_gate_proj(False, m) + + return route_gate, route_select_experts, moe_up, moe_act, moe_down + + +class Qwen3MoeModel(torch.nn.Module): + def __init__(self, args=None): + super(Qwen3MoeModel, self).__init__() + self.time_list = {} + self.args = args + # self.Embedding = Qwen3Embedding(self.args) + self.Attention = Qwen3MoeAttention(self.args) + # self.MLP = Qwen3MoeMLP(self.args) # TODO add MLP only + self.MoE = Qwen3MoeSparseMoeBlock(self.args) + self.aiob_forward_loops = getattr(args, "aiob_forward_loops", 10) + + def forward(self): + for i in range(self.aiob_forward_loops): + pre_norm, atten_qkv, atten_rotary_emb, atten_core, atten_o, post_norm = self.Attention() + self.time_list.setdefault("atten_norm", []).append( + {"time_gpu": pre_norm* 1e6} + ) + self.time_list.setdefault("atten_qkv", []).append( + {"time_gpu": atten_qkv* 1e6} + ) + self.time_list.setdefault("atten_rotary_emb", []).append( + {"time_gpu": atten_rotary_emb* 1e6} + ) + self.time_list.setdefault("atten_flash", []).append( + {"time_gpu": atten_core* 1e6} + ) + self.time_list.setdefault("atten_o", []).append( + {"time_gpu": atten_o* 1e6} + ) + self.time_list.setdefault("moe_norm", []).append( + {"time_gpu": post_norm* 1e6} + ) + moe_route_gate, moe_route_select_experts, moe_up, moe_act, moe_down = self.MoE() + self.time_list.setdefault("moe_route_gate", []).append( + {"time_gpu": moe_route_gate* 1e6} + ) + self.time_list.setdefault("moe_route_select_experts", []).append( + {"time_gpu": moe_route_select_experts* 1e6} + ) + self.time_list.setdefault("moe_expert_up", []).append( + {"time_gpu": moe_up* 1e6} + ) + self.time_list.setdefault("moe_expert_act", []).append( + {"time_gpu": moe_act* 1e6} + ) + self.time_list.setdefault("moe_expert_down", []).append( + {"time_gpu": moe_down* 1e6} + ) + print(self.time_list) + result_dir = "./results/aiob_outputs" + if not os.path.isdir(result_dir): + os.makedirs(result_dir) + filename = f"{self.args.model_name}_time_list_stats.txt" + file_name = os.path.join(result_dir, filename) + calculate_stats(self.time_list, file_name) + filepath = write_time(self.time_list, self.args) + process_all_keys(filepath) + return filepath + + +if __name__ == "__main__": + args = MockedQwen3Moe.Qwen3MoeParams() + x = torch.randint(0, args.vocab_size, (2, 1)) + model = Qwen3MoeModel(args) + filepath = model(x) + print(filepath) \ No newline at end of file diff --git a/aicb/workload_generator/mocked_model/inference/AiobQwen3Next.py b/aicb/workload_generator/mocked_model/inference/AiobQwen3Next.py new file mode 100644 index 000000000..dc013133e --- /dev/null +++ b/aicb/workload_generator/mocked_model/inference/AiobQwen3Next.py @@ -0,0 +1,698 @@ +import torch +# torch.set_default_device("cuda") +import workload_generator.mocked_model.inference.MockedQwen3Next as MockedQwen3Next +from workload_generator.mocked_model.MockedModel import InferencePhase +from utils.utils import * +from utils.deepgemm_utils import * +import torch.nn.functional as F +import random +from typing import Optional, Union, Tuple +# import triton +import numpy as np +from vllm.model_executor.layers.quantization.utils.fp8_utils import per_token_group_quant_fp8 +from vllm import _custom_ops as vllm_ops +from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding, get_rope +from vllm.model_executor.layers.fused_moe.fused_moe import fused_topk +from vllm.model_executor.layers.layernorm import GemmaRMSNorm +import flashinfer +from vllm.model_executor.layers.fla.ops import (RMSNormGated, fused_recurrent_gated_delta_rule) +from vllm.model_executor.layers.mamba.ops.causal_conv1d import ( + causal_conv1d_fn, causal_conv1d_update) +from vllm.platforms import current_platform +from vllm.triton_utils import tl, triton + +def GemmaRMSNormTest( + hidden_size, + variance_epsilon: float, + x: torch.Tensor, + residual: Optional[torch.Tensor]): + gemmaRMSNormImpl = GemmaRMSNorm(hidden_size, variance_epsilon) + gemmaRMSNormImpl = gemmaRMSNormImpl.to(x.device) + def test_func(): + gemmaRMSNormImpl.forward_cuda(x, residual) + t = bench_kineto(test_func, "triton_red_fused__to_copy_add_mean_mul_pow_rsqrt_0", suppress_kineto_output=True) + return t + +# modified from test_causal_conv1d.py +def test_causal_conv1d_update(batch, dim, width, seqlen, has_bias, silu_activation, + itype): + device = "cuda" + rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (3e-3, 5e-3) + if itype == torch.bfloat16: + rtol, atol = 1e-2, 5e-2 + # set seed + current_platform.seed_everything(0) + x = torch.randn(batch, dim, seqlen, device=device, dtype=itype) + x_ref = x.clone() + conv_state = torch.randn(batch, dim, width - 1, device=device, dtype=itype) + + weight = torch.randn(dim, width, device=device, dtype=itype) + bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None + activation = None if not silu_activation else "silu" + out = causal_conv1d_update(x, + conv_state, + weight, + bias, + activation=activation) + return out + +# modified from qwen3_next.py +# g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias) +@triton.jit +def fused_gdn_gating_kernel( + g, + A_log, + a, + dt_bias, + seq_len, + NUM_HEADS: tl.constexpr, + beta: tl.constexpr = 1.0, + threshold: tl.constexpr = 20.0, + BLK_HEADS: tl.constexpr = 8 +): + i_b, i_s, i_d = tl.program_id(0), tl.program_id(1), tl.program_id(2) + head_off = i_d * BLK_HEADS + tl.arange(0, BLK_HEADS) + off = i_b * seq_len * NUM_HEADS + i_s * NUM_HEADS + head_off + mask = head_off < NUM_HEADS + blk_A_log = tl.load(A_log + head_off, mask=mask) + blk_a = tl.load(a + off, mask=mask) + blk_bias = tl.load(dt_bias + head_off, mask=mask) + # If the model is loaded in fp16, without the .float() here, A might be -inf + x = blk_a.to(tl.float32) + blk_bias.to(tl.float32) + softplus_x = (1 / beta) * tl.log(1 + tl.exp(beta * x)) + blk_g = -tl.exp(blk_A_log.to(tl.float32)) * softplus_x + tl.store(g + off, blk_g.to(g.dtype.element_ty), mask=mask) + + +class Qwen3NextGatedDeltaNet(torch.nn.Module): + def __init__(self, args=None): + super(Qwen3NextGatedDeltaNet, self).__init__() + self.args = args + self.tp = self.args.tensor_model_parallel_size + + self.norm = RMSNormGated( + self.args.linear_value_head_dim, + eps=self.args.rms_norm_eps, + group_size=None, + norm_before_gate=True, + device='cuda', dtype=torch.bfloat16 + ) + + def _attn_quant(self, qk_or_o, m): + if qk_or_o: + n = self.args.hidden_size + else: + n = self.args.num_attention_heads * self.args.head_dim // self.tp + input = torch.randn(m, n, device='cuda', dtype=torch.bfloat16) + def test_func(): + per_token_group_quant_fp8(input, 128, column_major_scales=True) + + candidates = [ + 'per_token_group_quant_fp8', # old version + 'per_token_group_quant_8bit', # new version + ] + last_err = None + for names in candidates: + try: + t = bench_kineto(test_func, names, suppress_kineto_output=True) + return t + except Exception as e: + last_err = e + continue + + raise RuntimeError(f"bench_kineto failed for all candidates: {last_err}") + + def _attn_proj(self, k, n, m): + x_fp8, y_fp8, c, out, ref_out = construct(m, k, n) + # deep_gemm.gemm_fp8_fp8_bf16_nt(x_fp8, y_fp8, out) + deep_gemm.fp8_gemm_nt(x_fp8, y_fp8, out, c=c, disable_ue8m0_cast=True, recipe = None) + diff = calc_diff(out, ref_out) + assert diff < 0.001, f'{m=}, {k=}, {n=}, {diff:.5f}' + x_fp8, y_fp8, c, out, ref_out = construct(m, k, n) + def test_func(): + # deep_gemm.gemm_fp8_fp8_bf16_nt(x_fp8, y_fp8, out) + deep_gemm.fp8_gemm_nt(x_fp8, y_fp8, out, c=c, disable_ue8m0_cast=True, recipe = None) + t = bench_kineto(test_func, 'fp8_gemm', suppress_kineto_output=True) + return t + + def _causal_conv(self, m): + n = (self.args.num_attention_heads * self.args.head_dim + self.args.linear_num_key_heads*self.args.linear_key_head_dim + self.args.linear_num_value_heads*self.args.linear_value_head_dim) // self.tp + + def test_func(): + test_causal_conv1d_update(batch=m, + dim=n, + width=self.args.linear_conv_kernel_dim, + seqlen=1, # TODO decoding self.args.seq_length + has_bias=False, silu_activation=True, itype=torch.bfloat16) + t = bench_kineto(test_func, "_causal_conv1d_update_kernel", suppress_kineto_output=True) + return t + + def _fused_gdn_gating(self, m): + A_log = torch.empty( + divide(self.args.linear_num_value_heads, self.tp), + dtype=torch.float32, + ) + a = torch.randn(m, self.args.linear_num_value_heads//self.tp, device='cuda', dtype=torch.bfloat16) + dt_bias = torch.ones(self.args.linear_num_value_heads // self.tp) + + batch, num_heads = a.shape + seq_len=1 # TODO decoding self.args.seq_length + grid = (batch, seq_len, triton.cdiv(num_heads, 8)) + g = torch.empty_like(a, dtype=torch.float32) + + def test_func(): + fused_gdn_gating_kernel[grid](g, + A_log, + a, + dt_bias, + seq_len, + num_heads, + 1.0, + 20.0, + 8, + num_warps=1) + t = bench_kineto(test_func, "fused_gdn_gating_kernel", suppress_kineto_output=True) + return t + + def _fused_gdn_core(self): + phase = getattr(self.args, "phase", InferencePhase.DECODE.value) + if phase == InferencePhase.DECODE.value: + B = self.args.micro_batch + T = 1 + elif phase == InferencePhase.PREFILL.value: + B = 1 + T = self.args.seq_length + + H = self.args.linear_num_key_heads // self.tp + HV = self.args.linear_num_value_heads // self.tp + K = self.args.linear_key_head_dim + V = self.args.linear_value_head_dim + + # 1 is for decoding + q = torch.randn(B, 1, H, K, device='cuda', dtype=torch.bfloat16).contiguous() + k = F.normalize(torch.randn(B, 1, H, K, device='cuda', dtype=torch.bfloat16), p=2, dim=-1).contiguous() + v = torch.randn(B, 1, HV, V, device='cuda', dtype=torch.bfloat16).contiguous() + g = F.logsigmoid(torch.rand(B, 1, HV, device='cuda', dtype=torch.bfloat16)).contiguous() + beta = torch.rand(B, 1, HV, device='cuda', dtype=torch.bfloat16).sigmoid().contiguous() + # h0 = torch.empty(B, HV, V, device=q.device, dtype=v.dtype) + h0 = torch.randn(B, HV, K, V, device='cuda', dtype=torch.bfloat16).contiguous() + + ssm = torch.zeros(B*T, device='cuda', dtype=torch.int32).contiguous() + + # print('q', q.shape, q.stride()) + # print('k', k.shape, k.stride()) + # print('v', v.shape, v.stride()) + # print('g', g.shape, g.stride()) + # print('beta', None if beta is None else beta.shape) + # print('init_ht', None if h0 is None else h0.shape) + # print('ssm', None if ssm is None else ssm.shape) + + def test_func(): + o, ht = fused_recurrent_gated_delta_rule( + q, k, v, g, beta, + initial_state=h0, + inplace_final_state=True, + # inplace_final_state=False, + ssm_state_indices=ssm, + # ssm_state_indices=None, + # use_qk_l2norm_in_kernel=False, + use_qk_l2norm_in_kernel=True, + ) + t = bench_kineto(test_func, "fused_recurrent_gated_delta_rule_fwd_kernel", suppress_kineto_output=True) + return t + + def _norm(self, m): + k = self.args.linear_num_value_heads // self.tp + n = self.args.linear_value_head_dim + + core_attn_out = torch.randn(m*k, n, device='cuda', dtype=torch.bfloat16) + z = torch.randn(m*k, n, device='cuda', dtype=torch.bfloat16) + + def test_func(): + self.norm(core_attn_out, z) + t = bench_kineto(test_func, "layer_norm_fwd_kernel", suppress_kineto_output=True) + return t + + def forward(self): + phase = getattr(self.args, "phase", InferencePhase.DECODE.value) + if phase == InferencePhase.DECODE.value: + m = self.args.micro_batch + elif phase == InferencePhase.PREFILL.value: + m = self.args.seq_length + + # 1. qkv + qkv_quant = self._attn_quant(True, m) + + qkvz_proj_k = self.args.hidden_size + qkvz_proj_n = (self.args.linear_value_head_dim+self.args.linear_value_head_dim)*2 // self.tp + qkvz_proj = self._attn_proj(qkvz_proj_k, qkvz_proj_n, m) + + ba_proj_k = self.args.hidden_size + ba_proj_n = self.args.linear_num_value_heads*2 // self.tp + ba_proj = self._attn_proj(ba_proj_k, ba_proj_n, m) + + attn_qkv = qkv_quant+qkvz_proj+ba_proj + + # 2. Causal Convolution + attn_causal_conv = self._causal_conv(m) + + # 3. Recurrent Attention + gdn_gating = self._fused_gdn_gating(m) + #fused_recurrent_gated_delta_rule + gdn_core = self._fused_gdn_core() + + attn_gdn = gdn_gating + gdn_core + + # 4. output + # attn_out_norm=RMSNormGated + + o_norm = self._norm(m) + o_quant = self._attn_quant(False, m) + o_proj_k = self.args.linear_value_head_dim // self.tp + o_proj_n = self.args.hidden_size + o_proj = self._attn_proj(o_proj_k,o_proj_n, m) + + attn_o = o_norm + o_quant + o_proj + + return attn_qkv, attn_causal_conv, attn_gdn, attn_o + + +class Qwen3NextAttention(torch.nn.Module): + def __init__(self, args=None): + super(Qwen3NextAttention, self).__init__() + self.args = args + self.tp = self.args.tensor_model_parallel_size + + def _attn_quant(self, qk_or_o, m): + if qk_or_o: + n = self.args.hidden_size + else: + n = self.args.num_attention_heads * self.args.head_dim // self.tp + input = torch.randn(m, n, device='cuda', dtype=torch.bfloat16) + def test_func(): + per_token_group_quant_fp8(input, 128, column_major_scales=True) + + candidates = [ + 'per_token_group_quant_fp8', # old version + 'per_token_group_quant_8bit', # new version + ] + last_err = None + for names in candidates: + try: + t = bench_kineto(test_func, names, suppress_kineto_output=True) + return t + except Exception as e: + last_err = e + continue + + raise RuntimeError(f"bench_kineto failed for all candidates: {last_err}") + + def _attn_proj(self, qk_or_o, m): + if qk_or_o: + n = (self.args.num_attention_heads + self.args.num_key_value_heads * 2) * self.args.head_dim // self.tp + k = self.args.hidden_size + else: + n = self.args.hidden_size + k = self.args.num_attention_heads * self.args.head_dim // self.tp + x_fp8, y_fp8, c, out, ref_out = construct(m, k, n) + # deep_gemm.gemm_fp8_fp8_bf16_nt(x_fp8, y_fp8, out) + deep_gemm.fp8_gemm_nt(x_fp8, y_fp8, out, c=c, disable_ue8m0_cast=True, recipe = None) + diff = calc_diff(out, ref_out) + assert diff < 0.001, f'{m=}, {k=}, {n=}, {diff:.5f}' + x_fp8, y_fp8, c, out, ref_out = construct(m, k, n) + def test_func(): + # deep_gemm.gemm_fp8_fp8_bf16_nt(x_fp8, y_fp8, out) + deep_gemm.fp8_gemm_nt(x_fp8, y_fp8, out, c=c, disable_ue8m0_cast=True, recipe = None) + t = bench_kineto(test_func, 'fp8_gemm', suppress_kineto_output=True) + return t + def _qk_norm(self, q_or_k, m): + head_dim = self.args.head_dim + if q_or_k: + n = self.args.num_attention_heads * head_dim // self.tp + else: + n = self.args.num_key_value_heads * head_dim // self.tp + x = torch.randn(m,n, device='cuda', dtype=torch.bfloat16) + x = x.view(*x.shape[:-1], x.shape[-1] // head_dim, head_dim) + residual = torch.rand_like(x, device='cuda', dtype=torch.bfloat16) + return GemmaRMSNormTest(head_dim,self.args.rms_norm_eps,x, residual) + + def _rotary_emb(self, m): + head_dim = self.args.head_dim + n_q = self.args.num_attention_heads * head_dim // self.tp + n_k = self.args.num_key_value_heads * head_dim // self.tp + + q = torch.randn(m, n_q, device='cuda', dtype=torch.bfloat16) + k = torch.randn(m, n_k, device='cuda', dtype=torch.bfloat16) + pos = torch.arange(0, m, device='cuda') + rotary_emb = get_rope( + head_dim, + rotary_dim=head_dim, + max_position=self.args.max_position_embeddings, + base=self.args.rope_theta, + rope_scaling=self.args.rope_scaling, + dual_chunk_attention_config = getattr(self.args, + "dual_chunk_attention_config", + None), + ) + def test_func(): + rotary_emb(pos, q, k) + + t = bench_kineto(test_func, 'rotary_embedding_kernel', suppress_kineto_output=True) + return t + + def _attn_core(self, kv_lens): + block_size = 32 + num_blocks = 32768 + + head_size = self.args.head_dim + num_query_heads = self.args.num_attention_heads // self.tp + num_kv_heads = self.args.num_key_value_heads// self.tp + dtype = torch.bfloat16 + + + # based on vllm:test_flashinfer_decode_with_paged_kv + torch.set_default_device("cuda") + num_seqs = len(kv_lens) + assert num_query_heads % num_kv_heads == 0 + max_kv_len = max(kv_lens) + scale = head_size**-0.5 + soft_cap = None + + query = torch.randn(num_seqs, num_query_heads, head_size, dtype=dtype) + + key_value_cache = torch.randn(num_blocks, + 2, + block_size, + num_kv_heads, + head_size, + dtype=dtype) + # key_cache = key_value_cache[:, 0, :, :, :].squeeze(1) + # value_cache = key_value_cache[:, 1, :, :, :].squeeze(1) + + max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size + block_tables = torch.randint(0, + num_blocks, + (num_seqs, max_num_blocks_per_seq), + dtype=torch.int32) + + kv_indptr = [0] + kv_indices = [] + kv_last_page_lens = [] + for i in range(num_seqs): + seq_len = kv_lens[i] + assert seq_len > 0 + num_blocks = (seq_len + block_size - 1) // block_size + kv_indices.extend(block_tables[i, :num_blocks]) + kv_indptr.append(kv_indptr[-1] + num_blocks) + kv_last_page_len = seq_len % block_size + if kv_last_page_len == 0: + kv_last_page_len = block_size + kv_last_page_lens.append(kv_last_page_len) + + kv_indptr = torch.tensor(kv_indptr, dtype=torch.int32) + kv_indices = torch.tensor(kv_indices, dtype=torch.int32) + kv_last_page_lens = torch.tensor(kv_last_page_lens, dtype=torch.int32) + + # 128 MB + workspace_buffer = torch.empty(128 * 1024 * 1024, dtype=torch.int8) + wrapper = flashinfer.\ + BatchDecodeWithPagedKVCacheWrapper(workspace_buffer, "NHD", + use_tensor_cores=( + (num_query_heads//num_kv_heads) > 4) + ) + wrapper.plan(kv_indptr, + kv_indices, + kv_last_page_lens, + num_query_heads, + num_kv_heads, + head_size, + block_size, + "NONE", + q_data_type=dtype, + kv_data_type=dtype, + logits_soft_cap=soft_cap) + + def test_func(): + wrapper.run(query, key_value_cache) + t = bench_kineto(test_func, 'BatchPrefillWithPagedKVCacheKernel', suppress_kineto_output=True) + return t + + def forward(self): + phase = getattr(self.args, "phase", InferencePhase.DECODE.value) + if phase == InferencePhase.DECODE.value: + m = self.args.micro_batch + kv_lens = [self.args.seq_length] * m + elif phase == InferencePhase.PREFILL.value: + m = self.args.seq_length + kv_lens = [self.args.seq_length] * 1 + + # qkv + qkv_quant = self._attn_quant(True,m) + qkv_proj = self._attn_proj(True,m) + q_norm = self._qk_norm(True,m) + k_norm = self._qk_norm(False,m) + attn_qkv = qkv_quant + qkv_proj + q_norm + k_norm + + # rotary_embedding + rotary_emb = self._rotary_emb(m) + + # core + attn = self._attn_core(kv_lens) + + # o + o_quant = self._attn_quant(False,m) + o_proj = self._attn_proj(False,m) + attn_o = o_quant + o_proj + + return attn_qkv, rotary_emb, attn, attn_o + +class Qwen3NextSparseMoeBlock(torch.nn.Module): + def __init__(self, args): + super(Qwen3NextSparseMoeBlock, self).__init__() + self.args = args + self.batch_size = args.micro_batch + self.hidden_size = args.hidden_size + self.num_experts = args.num_experts + self.tp = args.tensor_model_parallel_size + self.dp = args.world_size // self.tp // args.pipeline_model_parallel + self.ep = self.tp * self.dp + self.topk = args.num_experts_per_tok + + def _route_gate(self, m): + # ReplicatedLinear + x = torch.randn(m, self.hidden_size, device='cuda', dtype=torch.bfloat16) + w = torch.randn(self.num_experts, self.hidden_size, device='cuda', dtype=torch.bfloat16) + # print(x.shape, w.shape) + + def test_func(): + F.linear(x, w) + + candidates = [ + ("cutlass", "cublasLt::splitK"), + ("nvjet_tst", "cublasLt::splitK"), + "cutlass", + "nvjet_tst", + ] + last_err = None + for names in candidates: + try: + res = bench_kineto(test_func, names, suppress_kineto_output=True) + return sum(res) if isinstance(res, (list, tuple)) else float(res) + except Exception as e: + last_err = e + continue + + raise RuntimeError(f"bench_kineto failed for all candidates: {last_err}") + + def _select_experts(self, total_tokens): + # FusedMoE.select_experts + hidden_states = torch.randn(total_tokens, self.hidden_size, device='cuda', dtype=torch.bfloat16) + router_logits = torch.randn(total_tokens, self.num_experts, device='cuda', dtype=torch.bfloat16) + + def test_func(): + fused_topk(hidden_states=hidden_states, + gating_output=router_logits, + topk=self.topk, + renormalize=True) + t = bench_kineto(test_func, 'moe::topkGatingSoftmax', suppress_kineto_output=True) + return t + + def _moe_gate_proj(self, up_or_down, m): + num_groups = self.num_experts // self.ep + strategy = getattr(self.args, "moe_routing_strategy", Strategy.RoundRobin) + expected_m_per_group = get_ep_expected_m_per_group( + m, num_groups, self.topk, self.ep, strategy + ) + if up_or_down: + n = 2 * self.args.moe_intermediate_size + k = self.hidden_size + else: + n = self.hidden_size + k = self.args.moe_intermediate_size + + phase = getattr(self.args, "phase", InferencePhase.DECODE.value) + + if phase == InferencePhase.DECODE.value: + t = bench_masked(num_groups, expected_m_per_group, k, n) + elif phase == InferencePhase.PREFILL.value: + t = bench_contiguous(num_groups, expected_m_per_group, k, n) + + return t + + def _moe_act(self, m): + # SiluAndMul + num_groups=self.num_experts // self.ep + total = m * self.dp * self.topk // self.num_experts + n = 2 * self.args.moe_intermediate_size + + total_max = total # TODO: maybe should set in config + intermediate_cache1 = torch.randn((num_groups * total_max, n), + device='cuda', + dtype=torch.bfloat16) + intermediate_cache2 = torch.randn((num_groups * total_max, n // 2), + device='cuda', + dtype=torch.bfloat16) + + def test_func(): + torch.ops._C.silu_and_mul( + intermediate_cache2.unflatten(0, (num_groups, total_max)), + intermediate_cache1.unflatten(0, (num_groups, total_max))) + + + candidates = [ + "vllm::silu", + "vectorized_elementwise_kernel", + ] + last_err = None + for names in candidates: + try: + res = bench_kineto(test_func, names, suppress_kineto_output=True) + return sum(res) if isinstance(res, (list, tuple)) else float(res) + except Exception as e: + last_err = e + continue + + raise RuntimeError(f"bench_kineto failed for all candidates: {last_err}") + + + def forward(self): + phase = getattr(self.args, "phase", InferencePhase.DECODE.value) + if phase == InferencePhase.DECODE.value: + m = self.args.micro_batch + total_tokens = m # TODO: maybe should set output seq_length + elif phase == InferencePhase.PREFILL.value: + m = self.args.seq_length + total_tokens = m + + route_gate = self._route_gate(m) + route_select_experts = self._select_experts(total_tokens) + moe_up = self._moe_gate_proj(True, m) + moe_act = self._moe_act(m) + moe_down = self._moe_gate_proj(False, m) + + return route_gate, route_select_experts, moe_up, moe_act, moe_down + + +class Qwen3NextModel(torch.nn.Module): + def __init__(self, args=None): + super(Qwen3NextModel, self).__init__() + self.time_list = {} + self.args = args + # self.Embedding = Qwen3Embedding(self.args) + self.FullAttention = Qwen3NextAttention(self.args) + self.GDN = Qwen3NextGatedDeltaNet(self.args) + # self.MLP = Qwen3NextMLP(self.args) # TODO add MLP only + self.MoE = Qwen3NextSparseMoeBlock(self.args) + self.aiob_forward_loops = getattr(args, "aiob_forward_loops", 10) + + def _norm(self): + phase = getattr(self.args, "phase", InferencePhase.DECODE.value) + if phase == InferencePhase.DECODE.value: + m = self.args.micro_batch + elif phase == InferencePhase.PREFILL.value: + m = self.args.seq_length + hidden_size = self.args.hidden_size + x = torch.randn(m,hidden_size, device='cuda', dtype=torch.bfloat16) + residual = torch.randn(m,hidden_size, device='cuda', dtype=torch.bfloat16) + return GemmaRMSNormTest(hidden_size, self.args.rms_norm_eps, x, residual) + + def forward(self): + for i in range(self.aiob_forward_loops): + # print(f'======================{i}======================') + # 1. input_layernorm + pre_norm = self._norm() + self.time_list.setdefault("atten_norm", []).append( + {"time_gpu": pre_norm* 1e6} + ) + # 2.A. self_attn + atten_qkv, atten_rotary_emb, atten_core, atten_o = self.FullAttention() + self.time_list.setdefault("atten_qkv", []).append( + {"time_gpu": atten_qkv* 1e6} + ) + self.time_list.setdefault("atten_rotary_emb", []).append( + {"time_gpu": atten_rotary_emb* 1e6} + ) + self.time_list.setdefault("atten_flash", []).append( + {"time_gpu": atten_core* 1e6} + ) + self.time_list.setdefault("atten_o", []).append( + {"time_gpu": atten_o* 1e6} + ) + + # 2.B. linear_attn + gdn_qkv, gdn_causal_conv, gdn_core, gdn_o = self.GDN() + self.time_list.setdefault("gdn_qkv", []).append( + {"time_gpu": gdn_qkv * 1e6} + ) + self.time_list.setdefault("gdn_causal_conv", []).append( + {"time_gpu": gdn_causal_conv * 1e6} + ) + self.time_list.setdefault("gdn_core", []).append( + {"time_gpu": gdn_core * 1e6} + ) + self.time_list.setdefault("gdn_o", []).append( + {"time_gpu": gdn_o * 1e6} + ) + + # 3. post_attention_layernorm + post_norm = self._norm() + self.time_list.setdefault("moe_norm", []).append( + {"time_gpu": post_norm* 1e6} + ) + + # 4. mlp + moe_route_gate, moe_route_select_experts, moe_up, moe_act, moe_down = self.MoE() + self.time_list.setdefault("moe_route_gate", []).append( + {"time_gpu": moe_route_gate* 1e6} + ) + self.time_list.setdefault("moe_route_select_experts", []).append( + {"time_gpu": moe_route_select_experts* 1e6} + ) + self.time_list.setdefault("moe_expert_up", []).append( + {"time_gpu": moe_up* 1e6} + ) + self.time_list.setdefault("moe_expert_act", []).append( + {"time_gpu": moe_act* 1e6} + ) + self.time_list.setdefault("moe_expert_down", []).append( + {"time_gpu": moe_down* 1e6} + ) + + print(self.time_list) + result_dir = "./results/aiob_outputs" + if not os.path.isdir(result_dir): + os.makedirs(result_dir) + filename = f"{self.args.model_name}_time_list_stats.txt" + file_name = os.path.join(result_dir, filename) + calculate_stats(self.time_list, file_name) + filepath = write_time(self.time_list, self.args) + process_all_keys(filepath) + return filepath + + +if __name__ == "__main__": + args = MockedQwen3Next.Qwen3NextParams() + x = torch.randint(0, args.vocab_size, (2, 1)) + model = Qwen3NextModel(args) + filepath = model(x) + print(filepath) \ No newline at end of file diff --git a/aicb/workload_generator/mocked_model/inference/MockedDeepSeek.py b/aicb/workload_generator/mocked_model/inference/MockedDeepSeek.py new file mode 100644 index 000000000..1d70ad5c0 --- /dev/null +++ b/aicb/workload_generator/mocked_model/inference/MockedDeepSeek.py @@ -0,0 +1,576 @@ +from utils.utils import divide, CommType, CommGroup +from workload_generator.mocked_model.MockedModel import MockedModel, Linear, MockedParam, MockedParamsBase +from log_analyzer.log import Workload, LogItem + +# multiplier to convert BF16 to (FP8 + FP32 scale) +# from https://github.com/deepseek-ai/DeepEP/blob/ef70b83e3b35a84aadc5385b02c95c5d1bcf299c/tests/test_internode.py#L194 +FP8_FACTOR = (1 + 4 / 128) / 2 + +class DeepSeekEmbedding(MockedModel): + def __init__( + self, + vocab_size, + hidden_size, + tp, + seq_len, + batch_size + ): + self.name = "embedding_layer" + self.layer_id = 0 + num_embedding_per_partition = divide(vocab_size, tp) + self.weight = MockedParam((num_embedding_per_partition, hidden_size), elem_size= 1) + self.tensor_model_parallel_size = tp + self.comm_size = 2 * batch_size * seq_len * hidden_size #allreduce + + def forward(self): + workloads = Workload() + if self.tensor_model_parallel_size > 1: + workloads.append( + LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.tp_group, + comm_group_size=self.tensor_model_parallel_size, + msg_size=self.comm_size, + stage="forward.DeepSeekEmbedding", + ) + ) + return workloads + + +class DeepSeekRowLinear(MockedModel): + def __init__( + self, + input_size, + output_size, + tp, + seq_len, + batch_size, + layer_id, + prefix_name, + # sequence_parallel_enabled=True, + computation_enable=False, + add_bias_linear=False, + elem_size = 1, + name=None, + ): + self.layer_id = layer_id + self.name = prefix_name + "_row" + self.input_size, self.output_size = input_size, output_size + self.input_size_per_partition = divide(input_size, tp) + self.weight = MockedParam( + (output_size, self.input_size_per_partition),elem_size, name=name + ) + if add_bias_linear: + self.bias = MockedParam((output_size, 1), elem_size, name=self.name + "_bias") + self.computation_enable = computation_enable + self.tensor_model_parallel_size, self.seq_length, self.micro_batch = tp, seq_len, batch_size + self.comm_size = 2 * seq_len * batch_size * output_size #allreduce + + def forward(self): + workloads = Workload() + if self.computation_enable: + workloads.append( + LogItem( + comm_type=CommType.computation, + msg_size=( + (self.seq_length, self.micro_batch, self.input_size_per_partition), + (self.input_size_per_partition, self.output_size), + ), + stage="forward.DeepSeekRowLinear." + self.name, + ) + ) + if self.tensor_model_parallel_size > 1: + workloads.append( + LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.tp_group, + comm_group_size=self.tensor_model_parallel_size, + msg_size=self.comm_size, + stage="forward.DeepSeekRowLinear." + self.name, + ) + ) + return workloads + +class DeepSeekColumnLinear(MockedModel): + def __init__( + self, + input_size, + output_size, + tp, + seq_len, + batch_size, + layer_id, + prefix_name="", + # sequence_parallel_enabled=True, + computation_enable=False, + add_bias_linear=False, + elem_size = 1, + name=None, + ): + self.layer_id = layer_id + self.name = prefix_name + "_column" + self.input_size, self.output_size = input_size, output_size + self.output_size_per_partition = divide(output_size, tp) + self.weight = MockedParam( + # fp8, elem_size = 1 + (input_size , self.output_size_per_partition), elem_size, name=name + ) + if add_bias_linear: + self.bias = MockedParam( + (self.output_size_per_partition, 1), elem_size, name=self.name + "_bias" + ) + # self.sequence_parallel_enabled = sequence_parallel_enabled + self.computation_enable = computation_enable + self.tensor_model_parallel_size, self.seq_length, self.micro_batch = tp, seq_len, batch_size + # self.comm_size = 1 * seq_len * batch_size * input_size #fp8 + + def forward(self): + workloads = Workload() + if self.computation_enable: + workloads.append( + LogItem( + comm_type=CommType.computation, + msg_size=( + (self.seq_length, self.micro_batch, self.input_size), + (self.input_size, self.output_size_per_partition) + ), + stage="forward.DeepSeekColumnLinear." + self.name, + ) + ) + return workloads + + +class DeepSeekAttention(MockedModel): + def __init__( + self, + hidden_size, + tp, + head_num, + d_q_c, #1536 + d_kv_c, #512 + d_r, #64 + d_q, + d_kv, + seq_len, + batch_size, + layer_id, + computation_enable=False, + add_bias_linear=False, + elem_size = 1 + + ): + self.layer_id = layer_id + self.name = "attention_layer" + self.hidden_size = hidden_size + self.tp = tp + self.n_heads = head_num//tp + self.q_lora = d_q_c + self.kv_lora = d_kv_c + self.rope_dim = d_r, + self.q_head_dim = d_q, + self.kv_head_dim = d_kv, + self.w_qkv_c = DeepSeekColumnLinear( + hidden_size, + (d_q_c + d_kv_c + d_r), + 1, #不分tp + seq_len, + batch_size, + layer_id, + "attention", + computation_enable, + add_bias_linear=False, + elem_size = 1, + name="attention_colum" + ) + #TODO: 每个矩阵的权重没有表征 + # self.weight_a = MockedParam() + self.w_q_cr = DeepSeekColumnLinear( + d_q_c, + (head_num*(d_q + d_r)), + tp, #对head分TP + seq_len, + batch_size, + layer_id, + "attention", + computation_enable, + add_bias_linear=False, + elem_size = 1, + name="attention_colum" + ) + #TODO：此处对于softmax(QK) V 的计算感觉不能简单用 ColumnLinear来表示， + # 但是又占了计算时间，先放着不写，看Mocked—Computation里是否会添加 + + self.q_k_v = DeepSeekColumnLinear( + d_kv_c, + (head_num * 2 * d_kv), + tp, + seq_len, + batch_size, + layer_id, + "attention", + computation_enable, + add_bias_linear=False, + elem_size = 1, + name="attention_colum" + ) + + self.wo = DeepSeekRowLinear( + (head_num * d_kv), + hidden_size, + tp, + seq_len, + batch_size, + layer_id, + "attention", + # sequence_parallel_enabled=True, + computation_enable, + add_bias_linear=False, + elem_size = 1, + name="attention_row", + ) + def forward(self): + workloads = Workload() + workloads.extend(self.w_qkv_c.forward()) + workloads.extend(self.w_q_cr.forward()) + workloads.extend(self.q_k_v.forward()) + workloads.extend(self.wo.forward()) + assert all([isinstance(workload, LogItem) for workload in workloads.workload]) + return workloads + +class DeepSeekMLP(MockedModel): + def __init__( + self, + hidden_size, + expert_dim, #ffn_hidden_dim + tp, + seq_len, + batch_size, + layer_id, + # sequence_parallel_enabled, + computation_enable, + add_bias_linear = False, + elem_size = 1, + name="" + ): + self.layer_id = layer_id + self.name = name + self.w1 = DeepSeekColumnLinear( + hidden_size,expert_dim*2,tp,seq_len, + batch_size, layer_id, name,computation_enable,add_bias_linear, elem_size,name="mlp_column" + ) + self.w2 = DeepSeekRowLinear( + expert_dim,hidden_size,tp,seq_len, + batch_size, layer_id, name,computation_enable,add_bias_linear, elem_size,name="mlp_row") + def forward(self): + workloads = Workload() + workloads.extend(self.w1.forward()) + workloads.extend(self.w2.forward()) + assert all([isinstance(workload, LogItem) for workload in workloads.workload]) + return workloads + +class DeepSeekMOE(MockedModel): + def __init__( + self, + hidden_size, + total_experts, + expert_model_parallel_size, + experts_topk, + expert_dim, + tp, + seq_len, + batch_size, + id, + shared_experts_cnt, + computation_enable, + add_bias_linear, + elem_size + ): + self.tp = tp + self.name = "sparse_moelayer" + self.layer_id = id + num_local_experts = total_experts // expert_model_parallel_size + # fc1_output_size = expert_dim * num_local_experts + # fc1_output_size_per_parttition = divide(fc1_output_size, tp) + # fc2_input_size = expert_dim * num_local_experts + # fc2_input_size_per_parttition = divide(fc2_input_size, tp) + # self.weight1 = MockedParam((hidden_size, fc1_output_size_per_parttition)) + # self.weight2 = MockedParam((fc2_input_size_per_parttition, hidden_size)) + # self.tensor_model_parallel_size = tp + self.expert_model_parallel_size = expert_model_parallel_size + self.topk = experts_topk + self.seq_length = seq_len + self.num_experts = total_experts + self.micro_batch = batch_size + self.hidden_size = hidden_size + self.shared_experts = [] + + self.w_gate = MockedParam((self.num_experts, hidden_size),elem_size,name = "moe_gate") + + self.expert = DeepSeekMLP( + hidden_size, + num_local_experts * expert_dim, + 1, #不分tp + seq_len, + batch_size, + id, + # sequence_parallel_enabled, + computation_enable, + add_bias_linear = False, + elem_size = 1, + name = "moe_expert" + ) + + for i in range(shared_experts_cnt): + self.shared_experts.append( + DeepSeekMLP( + hidden_size, + expert_dim, #ffn_hidden_dim + 1, #也没分TP + seq_len, + batch_size, + id, + # sequence_parallel_enabled, + computation_enable, + add_bias_linear = False, + name = "shared_experts" + ) + ) + def dispatch(self): + workloads = Workload() + workloads.append( + LogItem( + comm_type=CommType.all_to_all, + comm_group=CommGroup.ep_group, + comm_group_size= self.expert_model_parallel_size, + msg_size=self.seq_length + * self.hidden_size + * self.micro_batch + * (self.topk-1) + * 2 * FP8_FACTOR, + stage = "moe.dispatch" + ) + ) + return workloads + def combine(self): + workloads = Workload() + workloads.append( + LogItem( + comm_type=CommType.all_to_all, + comm_group=CommGroup.ep_group, + comm_group_size= self.expert_model_parallel_size, + msg_size=self.seq_length + * self.hidden_size + * self.micro_batch + * (self.topk-1) + * 2, + stage = "moe.combine" + ) + ) + return workloads + def forward(self): + workloads = Workload() + for shared_expert in self.shared_experts: + workloads.extend(shared_expert.forward()) + workloads.extend(self.dispatch()) + workloads.extend(self.expert.forward()) + workloads.extend(self.combine()) + return workloads + + +class DeepSeekRMSNorm(MockedModel): + def __init__(self, hidden_dim): + self.hidden_dim = hidden_dim + self.name = "RMSNorm" + self.weight = MockedParam((hidden_dim,1),1, name = self.name) + + +class DeepSeekTransformerLayer(MockedModel): + def __init__( + self, + dense_layer, + layerid, + hidden_size, + total_experts, + expert_model_parallel_size, + experts_topk, + expert_dim, + tp, + seq_len, + batch_size, + head_num, + d_q_c, #1536 + d_kv_c, #512 + d_r, #128 + d_q, + d_kv, + shared_experts_cnt, + # sequence_parallel_enabled=True, + computation_enable=False, + add_bias_linear=False, + elem_size = 1 + ): + # self.atten_norm = DeepSeekRMSNorm(hidden_size) + self.attention = DeepSeekAttention( + hidden_size, + tp, + head_num, + d_q_c, #1536 + d_kv_c, #512 + d_r, #128 + d_q, + d_kv, + seq_len, + batch_size, + layerid, + computation_enable=True, + add_bias_linear=False, + elem_size = 1, + ) + # self.ffn_norm = DeepSeekRMSNorm(hidden_size) + self.id = layerid + self.dense_layer = dense_layer + #不知道MockedParam这个类dpsk中是否有用 + if layerid < dense_layer: + self.mlp = DeepSeekMLP( + hidden_size, + expert_dim, #ffn_hidden_dim + tp, + seq_len, + batch_size, + layerid, + # sequence_parallel_enabled, + computation_enable, + # add_bias_linear + name = "dense_mlp" + ) + else: + self.mlp = DeepSeekMOE( + hidden_size, + total_experts, + expert_model_parallel_size, + experts_topk, + expert_dim, + tp, + seq_len, + batch_size, + layerid, + shared_experts_cnt, + # sequence_parallel_enabled, + computation_enable, + add_bias_linear, + elem_size=1 + ) + def forward(self): + workloads = Workload() + workloads.extend(self.attention.forward()) + workloads.extend(self.mlp.forward()) + assert all([isinstance(workload, LogItem) for workload in workloads.workload]) + return workloads + + +class DeepSeekModel(MockedModel): + def __init__(self, config): + #embedding层： + # self.embedding = DeepSeekEmbedding( + # config.vocab_size, + # config.hidden_size, + # config.tensor_model_parallel_size, + # config.seq_length, + # config.micro_batch + # ) + + #Transformer层： + self.layers = [ + DeepSeekTransformerLayer( + config.dense_layer, + i, + config.hidden_size, + config.num_experts, + config.expert_model_parallel_size, + config.moe_router_topk, + config.expert_dim, + config.tensor_model_parallel_size, + config.seq_length, + config.micro_batch, + config.num_attention_heads, + config.d_q_c, #1536 + config.d_kv_c, #512 + config.d_r, #128 + config.d_q, + config.d_kv, + config.shared_experts, + computation_enable=config.computation_enable, + add_bias_linear=config.add_bias_linear, + elem_size = 1 + ) + for i in range(config.num_layers) + ] + + #Norm层： + # self.norm = DeepSeekRMSNorm(config.hidden_size) + + #Final_layer + self.final = DeepSeekColumnLinear( + config.hidden_size, + config.vocab_size, + config.tensor_model_parallel_size, + config.seq_length, + config.micro_batch, + 1, + "final", + # sequence_parallel_enabled=True, + computation_enable=config.computation_enable, + add_bias_linear=config.add_bias_linear, + elem_size = 1, + name=None, + ) + + def forward(self, config): + workloads = Workload() + # workloads.extend(self.embedding.forward()) + for layer in self.layers: + workloads.extend(layer.forward()) + # workloads.extend(self.norm.forward()) + workloads.extend(self.final.forward()) + if config.tensor_model_parallel_size > 1: + workloads.append( + LogItem( + comm_type=CommType.all_gather, + comm_group=CommGroup.tp_group, + comm_group_size=config.tensor_model_parallel_size, + msg_size=1 * config.micro_batch * config.vocab_size, # fp8, config.vocab_size应该是总的vocab_size(即分到每个的*tp) + stage="forward.final", + ) + ) + assert all([isinstance(workload, LogItem) for workload in workloads.workload]) + return workloads + + +class DeepSeekParams(MockedParamsBase): + def __init__(self, config_file=None, args=None): + # Initialize base class with default values + super().__init__("DeepSeek-671B", "DeepSeek", config_file, args) + + # Enable computation_enable + self.computation_enable = True + self.add_bias_linear = False + + # Recalculate total_experts if needed + if hasattr(self, 'router_expert') or hasattr(self, 'duped_expert'): + self.num_experts = self.router_expert + self.duped_expert + + +if __name__ == "__main__": + import sys + + # Check if a config file is provided as a command line argument + config_file = None + if len(sys.argv) > 1: + config_file = sys.argv[1] + + args = DeepSeekParams(config_file) + model = DeepSeekModel(args) + workloads = model.forward(args) + filename = "testdpsk-1.csv" + workloads.dump(filename) + print("Finish Model initialization") \ No newline at end of file diff --git a/aicb/workload_generator/mocked_model/inference/MockedQwen3Moe.py b/aicb/workload_generator/mocked_model/inference/MockedQwen3Moe.py new file mode 100644 index 000000000..c4f1a253e --- /dev/null +++ b/aicb/workload_generator/mocked_model/inference/MockedQwen3Moe.py @@ -0,0 +1,84 @@ +from utils.utils import divide, CommType, CommGroup +from workload_generator.mocked_model.MockedModel import MockedModel, Linear, MockedParam, MockedParamsBase +from log_analyzer.log import Workload, LogItem +#TODO support Workload + +class Qwen3MoeRMSNorm(MockedModel): + def __init__(self, + layerid, + prefix_name): + self.name = prefix_name + "norm" + self.layer_id = layerid + +class Qwen3MoeAttention(MockedModel): + def __init__(self, + layerid): + self.name = "attention_layer" + self.layer_id = layerid + +class Qwen3MoeRoute(MockedModel): + def __init__(self, + layerid): + self.name = "moe_route" + self.layer_id = layerid + +class Qwen3MoeExpert(MockedModel): + def __init__(self, + layerid): + self.name = "moe_expert" + self.layer_id = layerid + +class Qwen3MoeBlock(MockedModel): + def __init__(self, + layerid): + self.name = "moe_block" + self.layer_id = layerid + self.route = Qwen3MoeRoute(layerid) + self.moeGemm = Qwen3MoeExpert(layerid) + +class Qwen3MoeTransformerLayer(MockedModel): + def __init__(self, + layerid + ): + self.pre_norm = Qwen3MoeRMSNorm(layerid,prefix_name="attention_") + self.attention = Qwen3MoeAttention(layerid) + self.post_norm = Qwen3MoeRMSNorm(layerid,prefix_name="moe_") + self.MoE = Qwen3MoeBlock(layerid) + + +class Qwen3MoeModel(MockedModel): + def __init__(self, config): + self.hidden_size = config.hidden_size + self.topk = config.num_experts_per_tok + self.num_hidden_layers = config.num_hidden_layers + self.moe_intermediate_size = config.moe_intermediate_size + self.layers = [ + Qwen3MoeTransformerLayer( + i + ) + for i in range(config.num_hidden_layers) + ] + # print(config) + +class Qwen3MoeParams(MockedParamsBase): + def __init__(self, config_file=None, args=None): + # Initialize base class with default values + super().__init__("Qwen3-Moe-235B", "Qwen3-Moe", config_file, args) + + # Recalculate total_experts if needed + if hasattr(self, 'router_expert') or hasattr(self, 'duped_expert'): + self.num_experts = self.router_expert + self.duped_expert + + +if __name__ == "__main__": + import sys + # Check if a config file is provided as a command line argument + config_file = None + if len(sys.argv) > 1: + config_file = sys.argv[1] + args = Qwen3MoeParams(config_file) + model = Qwen3MoeModel(args) + # workloads = model.forward(args) + # filename = "testqwen3-1.csv" #TODO: add forward + # workloads.dump(filename) + # print("Finish Model initialization") \ No newline at end of file diff --git a/aicb/workload_generator/mocked_model/inference/MockedQwen3Next.py b/aicb/workload_generator/mocked_model/inference/MockedQwen3Next.py new file mode 100644 index 000000000..4166fe383 --- /dev/null +++ b/aicb/workload_generator/mocked_model/inference/MockedQwen3Next.py @@ -0,0 +1,93 @@ +from utils.utils import divide, CommType, CommGroup +from workload_generator.mocked_model.MockedModel import MockedModel, Linear, MockedParam, MockedParamsBase +from log_analyzer.log import Workload, LogItem +#TODO support Workload + +class Qwen3NextRMSNorm(MockedModel): + def __init__(self, + layerid, + prefix_name): + self.name = prefix_name + "norm" + self.layer_id = layerid + +class Qwen3NextGatedDeltaNet(MockedModel): + def __init__(self, + layerid): + self.name = "attention_gdn" + self.layer_id = layerid + +class Qwen3NextAttention(MockedModel): + def __init__(self, + layerid): + self.name = "attention_layer" + self.layer_id = layerid + +class Qwen3NextRoute(MockedModel): + def __init__(self, + layerid): + self.name = "moe_route" + self.layer_id = layerid + +class Qwen3NextExpert(MockedModel): + def __init__(self, + layerid): + self.name = "moe_expert" + self.layer_id = layerid + +class Qwen3NextBlock(MockedModel): + def __init__(self, + layerid): + self.name = "moe_block" + self.layer_id = layerid + self.route = Qwen3NextRoute(layerid) + self.moeGemm = Qwen3NextExpert(layerid) + +class Qwen3NextTransformerLayer(MockedModel): + def __init__(self, + layerid, full_attention_flag + ): + self.pre_norm = Qwen3NextRMSNorm(layerid,prefix_name="attention_") + if full_attention_flag == 0: + self.attention = Qwen3NextAttention(layerid) + else: + self.attention = Qwen3NextGatedDeltaNet(layerid) + self.post_norm = Qwen3NextRMSNorm(layerid,prefix_name="moe_") + self.MoE = Qwen3NextBlock(layerid) + + +class Qwen3NextModel(MockedModel): + def __init__(self, config): + self.hidden_size = config.hidden_size + self.topk = config.num_experts_per_tok + self.num_hidden_layers = config.num_hidden_layers + self.moe_intermediate_size = config.moe_intermediate_size + self.layers = [ + Qwen3NextTransformerLayer( + i, (i+1)%config.full_attention_interval + ) + for i in range(config.num_hidden_layers) + ] + # print(config) + +class Qwen3NextParams(MockedParamsBase): + def __init__(self, config_file=None, args=None): + # Initialize base class with default values + super().__init__("Qwen3-Next-80B", "Qwen3-Next", config_file, args) + + # Recalculate total_experts if needed + if hasattr(self, 'router_expert') or hasattr(self, 'duped_expert'): + self.num_experts = self.router_expert + self.duped_expert + + +if __name__ == "__main__": + import sys + # Check if a config file is provided as a command line argument + config_file = None + if len(sys.argv) > 1: + config_file = sys.argv[1] + args = Qwen3NextParams(config_file) + model = Qwen3NextModel(args) + # workloads = model.forward(args) + # filename = "testqwen3-1.csv" #TODO: add forward + # workloads.dump(filename) + # print("Finish Model initialization") \ No newline at end of file diff --git a/aicb/workload_generator/mocked_model/inference/__init__.py b/aicb/workload_generator/mocked_model/inference/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/aicb/workload_generator/mocked_model/training/AiobDeepSeek.py b/aicb/workload_generator/mocked_model/training/AiobDeepSeek.py new file mode 100644 index 000000000..f2f70a14f --- /dev/null +++ b/aicb/workload_generator/mocked_model/training/AiobDeepSeek.py @@ -0,0 +1,917 @@ +"""Provides DeepSeek implementation for Aiob compute time calculations. + +Based on https://github.com/deepseek-ai/DeepSeek-V3/tree/f6e34dd26772dd4a216be94a8899276c5dca9e43 +and https://github.com/deepseek-ai/DeepGEMM/tree/391755ada0ffefa9a6a52b6f14dcaf22d1a463e0 + +Note: DeepGEMM sadly only works with SM90, SM100 i.e. Hopper and Blackwell + +Todo: +- add FlashMLA from https://github.com/deepseek-ai/FlashMLA/ for attentions + as of https://github.com/deepseek-ai/FlashMLA/commit/41b611f7d7561790a2f5040ff89212e08c7b0011 + FlashMLA's kernel supports backwards :) +- add DeepEP from https://github.com/deepseek-ai/DeepEP/ + +@misc{deepseekai2024deepseekv3technicalreport, + title={DeepSeek-V3 Technical Report}, + author={DeepSeek-AI}, + year={2024}, + eprint={2412.19437}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2412.19437}, +} + + +File: AiobDeepSeek.py +License: Apache 2.0 + +""" + +import random +from dataclasses import dataclass +from typing import Optional +import math + +import torch +import torch.nn.functional as F +import torch.nn as nn + +# from workload_generator.mocked_model.training.AiobMegatron import linear_with_grad_accumulation_and_async_allreduce +# can't import because of circular imports +# FIXME: maybe move this to utils + +from torch.cuda.amp import custom_bwd, custom_fwd +class LinearWithGradAccumulationAndAsyncCommunication(torch.autograd.Function): + """See linear_with_grad_accumulation_and_async_allreduce""" + + @staticmethod + @custom_fwd + def forward( + ctx, + input, + weight, + bias, + gradient_accumulation_fusion, + async_grad_allreduce, + sequence_parallel, + tp, + ): + ctx.save_for_backward(input, weight) # + ctx.use_bias = bias is not None + ctx.gradient_accumulation_fusion = gradient_accumulation_fusion + ctx.async_grad_allreduce = async_grad_allreduce + ctx.sequence_parallel = sequence_parallel + + if sequence_parallel: + + total_input = input + else: + total_input = input + + output = torch.matmul(total_input, weight.t()) + + if bias is not None: + output = output + bias + return output + + +def linear_with_grad_accumulation_and_async_allreduce( + input: torch.Tensor, + weight: torch.Tensor, + bias: Optional[torch.Tensor], + gradient_accumulation_fusion: bool, + async_grad_allreduce: bool, + sequence_parallel: bool, + tp, +) -> torch.Tensor: + """Linear layer execution with asynchronous communication and + gradient accumulation fusion in backprop. + + This has the option to accumulate the result of backprop + calculation into an existing gradient buffer, preventing the need + to do an additional addition kernel after the gradient + calculation. + + Additionally, the tensor parallel all reduce of the input + gradients can be done asynchronously with the calculation of + the weight gradients. + + In the case of sequence parallelism, the reduce scatter of the + input gradients is done asynchronously with the calcluation of the + weight gradients. + + Use of this module requires that the environment variable + CUDA_DEVICE_MAX_CONNECTIONS=1. There are a few collective + operations, noted in the code, that should be scheduled before + compute kernels to overlap the communication with the computation, + which is necessary for a speedup but not for correctness so that + ordering isn't imposed by the scheduler. Setting + CUDA_DEVICE_MAX_CONNECTIONS=1 forces the kernels to be scheduled + in the order they are called. + + Arguments: + + input (torch.Tensor required): input like torch.nn.functional.linear + + weight (torch.Tensor required): weight like torch.nn.functional.linear + + bias (torch.Tensor optional): bias like torch.nn.functional.linear + + gradient_accumulation_fusion (bool required): Perform the gradient + accumulation fusion, requires the custom CUDA extension + fused_weight_gradient_mlp_cuda module. To use + gradient_accumulation_fusion you must install APEX with + --cpp_ext and --cuda_ext. For example: "pip install + --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext .\" + " Note that the extension requires CUDA>=11. Otherwise, you + must turn off gradient accumulation fusion." + + async_grad_allreduce (bool required): Do the allreduce of input + gradients asyncronously with the computation of weight + gradients. If sequence_parallel is True, this must be + False, as no all reduce is performed. + + sequence_parallel (bool required): Indicates that sequence + parallelism is used and thus in the forward pass the input is + all gathered, and the backward pass the input gradients are + reduce scattered. + """ + args = [ + input, + weight, + bias, + gradient_accumulation_fusion, + async_grad_allreduce, + sequence_parallel, + tp, + ] + + return LinearWithGradAccumulationAndAsyncCommunication.apply(*args) + + +linear_with_grad_accumulation_and_async_allreduce.warned = False + +from utils.utils import cuda_timing_decorator, divide + +# needs deep_gemm from https://github.com/deepseek-ai/DeepGEMM +try: + import deep_gemm + from deep_gemm import ceil_div +except ImportError as e: + print(""" + Needs deep_gemm from https://github.com/deepseek-ai/DeepGEMM + to install, + git clone --recursive git@github.com:deepseek-ai/DeepGEMM.git + cd DeepGEMM + ./install.sh + """) + raise e + + +# from deepgemm/tests +def per_token_cast_to_fp8(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 + m, n = x.shape + pad_size = (128 - (n % 128)) % 128 + x = torch.nn.functional.pad( + x, (0, pad_size), value=0) if pad_size > 0 else x + x_view = x.view(m, -1, 128) + x_amax = x_view.abs().float().amax(dim=2).view(m, -1).clamp(1e-4) + fp8_data = (x_view * (448.0 / x_amax.unsqueeze(2))).to(torch.float8_e4m3fn) + return fp8_data.view(m, n + pad_size)[:, :n], (x_amax / 448.0).view(m, -1) + + +def per_block_cast_to_fp8(x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + assert x.dim() == 2 + m, n = x.shape + x_padded = torch.zeros( + (ceil_div(m, 128) * 128, ceil_div(n, 128) * 128), + dtype=x.dtype, + device=x.device, + ) + x_padded[:m, :n] = x + x_view = x_padded.view(-1, 128, x_padded.size(1) // 128, 128) + x_amax = x_view.abs().float().amax(dim=(1, 3), keepdim=True).clamp(1e-4) + x_scaled = (x_view * (448.0 / x_amax)).to(torch.float8_e4m3fn) + return x_scaled.view_as(x_padded)[:m, :n].contiguous(), (x_amax / 448.0).view( + x_view.size(0), + x_view.size(2), + ) + + +@dataclass +class Args: + hidden_size: int + ffn_hidden_size: int + micro_batch: int + expert_model_parallel_size: int + num_experts: int + moe_router_topk: int + seq_length: int + n_shared_expert: int + qk_nope_dim: int + qk_rope_dim: int + v_head_dim: int + q_lora_rank: int + kv_lora_rank: int + num_attention_heads: int + enable_sequence_parallel: bool + tensor_model_parallel_size: int + +# from deepseek (without quantization part) +# https://github.com/deepseek-ai/DeepSeek-V3/blob/9b4e9788e4a3a731f7567338ed15d3ec549ce03b/inference/model.py +def linear(x: torch.Tensor, weight: torch.Tensor, bias: Optional[torch.Tensor] = None) -> torch.Tensor: + """ + Applies a linear transformation to the incoming data: y = xA^T + b. + + Args: + x (torch.Tensor): The input tensor. + weight (torch.Tensor): The weight tensor. It may be quantized and + requires dequantization for certain cases. + bias (Optional[torch.Tensor]): The bias tensor to be added. Default is None. + """ + # sequence parallel, and tp are no-op in linear_with_grad_accumulation_and_async_allreduce -> LinearWithGradAccumulationAndAsyncCommunication + return linear_with_grad_accumulation_and_async_allreduce(input=x, + weight=weight, + bias=bias, + gradient_accumulation_fusion=True, + async_grad_allreduce=False, + sequence_parallel=True, + tp=1) + + +class Linear(nn.Module): + """ + Custom linear layer with support for quantized weights and optional bias. + + Args: + in_features (int): Number of input features. + out_features (int): Number of output features. + bias (bool): Whether to include a bias term. Defaults to False. + dtype (optional): Data type for the layer. Defaults to `torch.bfloat16`. + """ + dtype = torch.bfloat16 + + def __init__(self, in_features: int, out_features: int, bias: bool = False, dtype = None): + super().__init__() + self.in_features = in_features + self.out_features = out_features + self.weight = nn.Parameter(torch.empty(out_features, in_features, dtype=dtype or Linear.dtype)) + if self.weight.element_size() == 1: + pass + # skip scaling part + # scale_out_features = (out_features + block_size - 1) // block_size + # scale_in_features = (in_features + block_size - 1) // block_size + # self.weight.scale = self.scale = nn.Parameter(torch.empty(scale_out_features, scale_in_features, dtype=torch.float32)) + else: + self.register_parameter("scale", None) + if bias: + self.bias = nn.Parameter(torch.empty(out_features)) + else: + self.register_parameter("bias", None) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + Forward pass for the custom linear layer. + + Args: + x (torch.Tensor): Input tensor. + + Returns: + torch.Tensor: Transformed tensor after linear computation. + """ + return linear(x, self.weight, self.bias) + + +class ColumnParallelLinear(Linear): + """ + Linear layer with column parallelism, splitting output features across distributed processes. + + Args: + in_features (int): Number of input features. + out_features (int): Total number of output features. + bias (bool): Whether to include a bias term. Defaults to False. + dtype (optional): Data type for the layer. Defaults to `torch.bfloat16`. + """ + def __init__(self, in_features: int, out_features: int, bias: bool = False, dtype = None, world_size: int = 1): + assert out_features % world_size == 0, f"Output features must be divisible by world size (world_size={world_size})" + self.part_out_features = out_features // world_size + super().__init__(in_features, self.part_out_features, bias, dtype) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + Forward pass for column parallel linear layer. + + Args: + x (torch.Tensor): Input tensor. + + Returns: + torch.Tensor: Transformed tensor with column-parallel computation. + """ + y = linear(x, self.weight, self.bias) + return y + + +class RowParallelLinear(Linear): + """ + Linear layer with row parallelism, splitting input features across distributed processes. + + Args: + in_features (int): Total number of input features. + out_features (int): Number of output features. + bias (bool): Whether to include a bias term. Defaults to False. + dtype (optional): Data type for the layer. Defaults to `torch.bfloat16`. + """ + def __init__(self, in_features: int, out_features: int, bias: bool = False, dtype = None, world_size: int = 1): + assert in_features % world_size == 0, f"Input features must be divisible by world size (world_size={world_size})" + self.part_in_features = in_features // world_size + super().__init__(self.part_in_features, out_features, bias, dtype) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + Forward pass for row parallel linear layer. + + Args: + x (torch.Tensor): Input tensor. + + Returns: + torch.Tensor: Transformed tensor with row-parallel computation. + """ + y = linear(x, self.weight) + # if world_size > 1: + # dist.all_reduce(y) + if self.bias is not None: + y += self.bias + return y +class RMSNorm(torch.nn.Module): + """ + Root Mean Square Layer Normalization (RMSNorm). + + Args: + dim (int): Dimension of the input tensor. + eps (float): Epsilon value for numerical stability. Defaults to 1e-6. + """ + def __init__(self, dim: int, eps: float = 1e-6, dtype = torch.bfloat16): + super().__init__() + self.dim = dim + self.eps = eps + self.weight = torch.nn.Parameter(torch.ones(dim, dtype=dtype)) + + def forward(self, x: torch.Tensor): + """ + Forward pass for RMSNorm. + + Args: + x (torch.Tensor): Input tensor. + + Returns: + torch.Tensor: Normalized tensor with the same shape as input. + """ + return F.rms_norm(x, (self.dim,), self.weight, self.eps) + +def apply_rotary_emb(x: torch.Tensor, freqs_cis: torch.Tensor) -> torch.Tensor: + """ + Applies rotary positional embeddings to the input tensor. + + Args: + x (torch.Tensor): Input tensor with positional embeddings to be applied. + freqs_cis (torch.Tensor): Precomputed complex exponential values for positional embeddings. + + Returns: + torch.Tensor: Tensor with rotary embeddings applied. + """ + dtype = x.dtype + x = torch.view_as_complex(x.float().view(*x.shape[:-1], -1, 2)) + freqs_cis = freqs_cis.view(1, x.size(1), 1, x.size(-1)) + y = torch.view_as_real(x * freqs_cis).flatten(3) + return y.to(dtype) + +def precompute_freqs_cis(args: Args) -> torch.Tensor: + """ + Precomputes frequency-based complex exponential values for rotary positional embeddings. + + Args: + args (ModelArgs): Model arguments containing positional embedding parameters. + + Returns: + torch.Tensor: Precomputed complex exponential values for positional embeddings. + """ + dim = args.qk_rope_dim + seqlen = args.seq_length + beta_fast = 32 + beta_slow = 1 + base = 10000.0 + factor = 40 + + def find_correction_dim(num_rotations, dim, base, max_seq_len): + """ + Computes the correction dimension for a given number of rotations in the rotary positional embedding. + + Args: + num_rotations (float): Number of rotations to compute the correction for. + dim (int): Dimensionality of the embedding space. + base (float): Base value for the exponential computation. + max_seq_len (int): Maximum sequence length. + + Returns: + float: The correction dimension based on the input parameters. + """ + return dim * math.log(max_seq_len / (num_rotations * 2 * math.pi)) / (2 * math.log(base)) + + def find_correction_range(low_rot, high_rot, dim, base, max_seq_len): + """ + Computes the range of correction dimensions for rotary positional embeddings. + + Args: + low_rot (float): Lower bound for the number of rotations. + high_rot (float): Upper bound for the number of rotations. + dim (int): Dimensionality of the embedding space. + base (float): Base value for the exponential computation. + max_seq_len (int): Maximum sequence length. + + Returns: + Tuple[int, int]: The range of correction dimensions (low, high), clamped to valid indices. + """ + low = math.floor(find_correction_dim(low_rot, dim, base, max_seq_len)) + high = math.ceil(find_correction_dim(high_rot, dim, base, max_seq_len)) + return max(low, 0), min(high, dim-1) + + def linear_ramp_factor(min, max, dim): + """ + Computes a linear ramp function used to smooth values between a minimum and maximum range. + + Args: + min (float): Minimum value for the ramp function. + max (float): Maximum value for the ramp function. + dim (int): Dimensionality of the ramp tensor. + + Returns: + torch.Tensor: A tensor of shape (dim,) with values linearly interpolated between 0 and 1, + clamped to the range [0, 1]. + """ + if min == max: + max += 0.001 + linear_func = (torch.arange(dim, dtype=torch.float32) - min) / (max - min) + ramp_func = torch.clamp(linear_func, 0, 1) + return ramp_func + + freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) + if seqlen > 4096: + low, high = find_correction_range(beta_fast, beta_slow, dim, base, 4096) + smooth = 1 - linear_ramp_factor(low, high, dim // 2) + freqs = freqs / factor * (1 - smooth) + freqs * smooth + + t = torch.arange(seqlen) + freqs = torch.outer(t, freqs) + freqs_cis = torch.polar(torch.ones_like(freqs), freqs) + return freqs_cis + +############################# +# End of copy from DeepSeek # +############################# +class DeepSeekMLA(torch.nn.Module): + """DeepSeekMLA layer for AIOB + + Args: + torch (torch.Tensor): input tensor + + forward() return timing in format of dict, + i.e. + layer_time_map = { + "attention_linear_q_lora": q_lora_time, + "attention_q_column": q_time, + "attention_linear_kv_lora": kv_lora_time, + "attention_kv_column": kv_time, + "attention_o_row": o_time + } + # FIXME: create a generic/structured way to return timings + # FIXME: fix apply_pe functions, right now they're commented out, should be trivial compute overhead + """ + def __init__(self, args: Args): + super().__init__() + self.register_buffer("freqs_cis", precompute_freqs_cis(args), persistent=False) + self.args = args + self.enable_sequence_parallel = args.enable_sequence_parallel + self.tp = args.tensor_model_parallel_size + self.qk_head_dim = self.args.qk_nope_dim + self.args.qk_rope_dim + # init torch + self.device = torch.cuda.current_device() + self.dtype = torch.bfloat16 + self.hidden_size = self.args.hidden_size + self.n_local_heads = self.args.num_attention_heads // self.tp + + + # Q down projection + self.wq_a = Linear(self.args.hidden_size, self.args.q_lora_rank, dtype=self.dtype) + self.q_norm = RMSNorm(self.args.q_lora_rank, dtype=self.dtype) + + self.wq_b = ColumnParallelLinear( + self.args.q_lora_rank, self.args.num_attention_heads * self.qk_head_dim, + dtype=self.dtype, + world_size=self.tp + ) + + # # KV down projection + self.wkv_a = Linear( + self.args.hidden_size, self.args.kv_lora_rank + self.args.qk_rope_dim, dtype=self.dtype + ) + + self.kv_norm = RMSNorm(self.args.kv_lora_rank, dtype=self.dtype) + self.wkv_b = ColumnParallelLinear( + self.args.kv_lora_rank, + self.args.num_attention_heads + * (self.args.qk_nope_dim + self.args.v_head_dim), + dtype=self.dtype, + world_size=self.tp + ) + + self.wo = RowParallelLinear( + self.args.num_attention_heads * self.args.v_head_dim, self.args.hidden_size, + dtype=self.dtype, + world_size=self.tp + ) + + self.softmax_scale = self.qk_head_dim**-0.5 + + @cuda_timing_decorator + def _apply_linear_q_lora(self, x): + return self.q_norm(self.wq_a(x)) + + @cuda_timing_decorator + def _apply_q(self, x): + bsz, seqlen, _ = x.size() + q = self.wq_b(x) + q = q.view(bsz, seqlen, self.n_local_heads, self.qk_head_dim) + q_nope, q_pe = torch.split(q, [self.args.qk_nope_dim, self.args.qk_rope_dim], dim=-1) + # q_pe = apply_rotary_emb(q_pe, self.freqs_cis) + return (q_nope, q_pe) + + @cuda_timing_decorator + def _apply_linear_kv_lora(self, x): + kv = self.wkv_a(x) + kv, k_pe = torch.split(kv, [self.args.kv_lora_rank, self.args.qk_rope_dim], dim=-1) + kv = self.kv_norm(kv) + return (kv, k_pe) + + @cuda_timing_decorator + def _apply_kv(self, x, q_nope, q_pe, kv, k_pe): + + # k_pe = apply_rotary_emb(k_pe.unsqueeze(2), freqs_cis) + k_pe.unsqueeze(2) + + wkv_b = self.wkv_b.weight + wkv_b = wkv_b.view(self.n_local_heads, -1, self.args.kv_lora_rank) + + q_nope = torch.einsum("bshd,hdc->bshc", q_nope, wkv_b[:, :self.args.qk_nope_dim]) + + scores = (torch.einsum("bshc,btc->bsht", q_nope, kv) + + torch.einsum("bshr,btr->bsht", q_pe, k_pe.squeeze(2))) * self.softmax_scale + + scores = scores.softmax(dim=-1, dtype=torch.float32).type_as(x) + x = torch.einsum("bsht,btc->bshc", scores, kv) + x = torch.einsum("bshc,hdc->bshd", x, wkv_b[:, -self.args.v_head_dim:]) + return x + + @cuda_timing_decorator + def _apply_kv_naive(self, x, q_nope, q_pe): + bsz, seqlen, _ = x.size() + # compress kv + kv = self.wkv_a(x) + kv, k_pe = torch.split(kv, [self.args.kv_lora_rank, self.args.qk_rope_dim], dim=-1) + # rotate + # k_pe = apply_rotary_emb(k_pe.unsqueeze(2), self.freqs_cis) + k_pe = k_pe.unsqueeze(2) + + q = torch.cat([q_nope, q_pe], dim=-1) + kv = self.wkv_b(self.kv_norm(kv)) + kv = kv.view(bsz, seqlen, self.n_local_heads, self.args.qk_nope_dim + self.args.v_head_dim) + k_nope, v = torch.split(kv, [self.args.qk_nope_dim, self.args.v_head_dim], dim=-1) + + print(f"kv {kv.shape} k_pe: {k_pe.shape}") + k = torch.cat([k_nope, k_pe.expand(-1, -1, self.n_local_heads, -1)], dim=-1) + scores = torch.einsum("bshd,bthd->bsht", q, k) * self.softmax_scale + scores = scores.softmax(dim=-1, dtype=torch.float32).type_as(x) + x = torch.einsum("bsht,bthd->bshd", scores, v) + return x + + @cuda_timing_decorator + def _apply_wo(self, x): + x = self.wo(x.flatten(2)) + return x + + def forward(self, x): + q, q_lora_time = self._apply_linear_q_lora(x) + (q_nope, q_pe), q_time = self._apply_q(q) + (kv, k_pe), kv_lora_time = self._apply_linear_kv_lora(x) + x, kv_time = self._apply_kv(x, q_nope, q_pe, kv, k_pe) + x, o_time = self._apply_wo(x) + layer_time_map = { + "attention_linear_q_lora": q_lora_time, + "attention_q_column": q_time, + "attention_linear_kv_lora": kv_lora_time, + "attention_kv_column": kv_time, + "attention_o_row": o_time + } + return x, layer_time_map + +def test_deepseek_mla() -> torch.Tensor: + x = torch.randn(5120, 1, 12288, dtype=torch.bfloat16) + args = Args( + expert_model_parallel_size=1, + ffn_hidden_size=4096, + hidden_size=12288, + num_experts=8, + micro_batch=1, + moe_router_topk=4, + n_shared_expert=2, + seq_length=5120, + enable_sequence_parallel=True, + kv_lora_rank=512, + num_attention_heads= 128, + q_lora_rank=1536, + qk_nope_dim=128, + qk_rope_dim=64, + tensor_model_parallel_size=1, + v_head_dim=128 + ) + model = DeepSeekMLA(args) + y = model(x) + return y + +class DeepSeekExpert(torch.nn.Module): + """DeepSeekExpert""" + + def __init__(self, args: Args): + super().__init__() + self.hidden_size = args.hidden_size + self.ffn_hidden_size = args.ffn_hidden_size + self.dev = torch.cuda.current_device() + self.w1 = torch.randn( + args.ffn_hidden_size, + args.hidden_size, + device=self.dev, + ).to(torch.bfloat16) + self.w1_fp8 = per_block_cast_to_fp8(self.w1) + + self.w2 = torch.randn( + args.hidden_size, + args.ffn_hidden_size, + device=self.dev, + ).to(torch.bfloat16) + + self.w2_fp8 = per_block_cast_to_fp8(self.w2) + + self.w3 = torch.randn( + args.ffn_hidden_size, + args.hidden_size, + device=self.dev, + ).to(torch.bfloat16) + + self.w3_fp8 = per_block_cast_to_fp8(self.w3) + + self.o0 = torch.empty( + (args.seq_length, self.ffn_hidden_size), + device=self.dev, + dtype=torch.bfloat16, + ) + self.o1 = torch.empty( + (args.seq_length, self.ffn_hidden_size), + device=self.dev, + dtype=torch.bfloat16, + ) + self.out = torch.empty( + (args.seq_length, self.hidden_size), + device=self.dev, + dtype=torch.bfloat16, + ) + + @cuda_timing_decorator + def _apply_linear1(self, x: torch.Tensor, x_scale: torch.Tensor): + deep_gemm.gemm_fp8_fp8_bf16_nt((x, x_scale), self.w1_fp8, self.o0) + deep_gemm.gemm_fp8_fp8_bf16_nt((x, x_scale), self.w3_fp8, self.o1) + return (self.o0, self.o1) + + @cuda_timing_decorator + def _apply_linear2(self, x: torch.Tensor): + self.out = torch.matmul(x, self.w2.T) + # or one can scale it down to fp8 and apply deepgemm + # x_fp8 = per_token_cast_to_fp8(x) + # deep_gemm.gemm_fp8_fp8_bf16_nt(x_fp8, self.w2_fp8, self.out) + return self.out + + @cuda_timing_decorator + def _apply_activation(self, x: torch.Tensor): + return F.silu(x) + + def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, float, float, float]: + # based on https://github.com/deepseek-ai/DeepSeek-V3/blob/f6e34dd26772dd4a216be94a8899276c5dca9e43/inference/model.py#L630 + # i.e. out = self.w2(F.silu(self.w1(x)) * self.w3(x)) + x, x_scale = per_token_cast_to_fp8(x.view(-1, self.hidden_size)) + l1_out, l1_time = self._apply_linear1(x, x_scale) + act_out, act_time = self._apply_activation(l1_out[0]) + l2_out, l2_time = self._apply_linear2(act_out * l1_out[0]) + return l2_out, l1_time, act_time, l2_time + + +class DeepSeekMoE(torch.nn.Module): + """DeepSeekMoE""" + + def __init__(self, args: Args): + super().__init__() + self.n_local_experts = ( + args.n_shared_expert + args.num_experts // args.expert_model_parallel_size + ) + self.hidden_size = args.hidden_size + self.dev = torch.cuda.current_device() + in_dim = self.hidden_size * args.micro_batch + self.in_dim = in_dim + self.inter_dim = args.ffn_hidden_size * self.n_local_experts + # yoinked from AiobMegatron.py->MoELayer + ep = args.expert_model_parallel_size + num_experts = args.num_experts + micro_batch = args.micro_batch + seq_len = args.seq_length + topk = args.moe_router_topk + self.topk = topk + hidden_size = args.hidden_size + self.dispatched_input = torch.rand( + int(seq_len * micro_batch * topk * ep / + num_experts * self.n_local_experts), + hidden_size, + device=self.dev, + ).to(torch.bfloat16) + self.dispatched_input_fp8 = per_token_cast_to_fp8( + self.dispatched_input) + + self.tokens_per_expert = torch.full( + (self.n_local_experts,), + int(seq_len * micro_batch * topk * ep / num_experts), + ) + + self.local_experts = torch.nn.ModuleList() + for _ in range(self.n_local_experts): + self.local_experts.append(DeepSeekExpert(args)) + + def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, int, int, int]: + mlp_l1_all, mlp_act_all, mlp_l2_all = 0, 0, 0 + o = torch.zeros( + self.n_local_experts, + x.shape[0], + self.hidden_size, + device=self.dev, + ).to(torch.bfloat16) + for i, expert in enumerate(self.local_experts): + out, mlp_l1, mlp_act, mlp_l2 = expert(x.view(-1, self.in_dim)) + mlp_l1_all += mlp_l1 + mlp_act_all += mlp_act + mlp_l2_all += mlp_l2 + o[i] = out + return o, mlp_l1_all, mlp_act_all, mlp_l2_all + + +# some util tests to measure timings +def test_deepseek_expert() -> torch.Tensor: + x = torch.randn(2048, 1, 1024, dtype=torch.bfloat16) + args = Args( + expert_model_parallel_size=1, + ffn_hidden_size=4096, + hidden_size=1024, + num_experts=8, + micro_batch=1, + moe_router_topk=4, + n_shared_expert=2, + seq_length=2048, + enable_sequence_parallel=True, + kv_lora_rank=512, + num_attention_heads= 128 + 64, + q_lora_rank=0, + qk_nope_dim=128, + qk_rope_dim=64, + tensor_model_parallel_size=1, + v_head_dim=128 + ) + model = DeepSeekExpert(args) + y = model(x) + print(y) + + +def test_deepseek_mlp(): + x = torch.randn(2048, 1, 1024, dtype=torch.bfloat16) + args = Args( + expert_model_parallel_size=1, + ffn_hidden_size=4096, + hidden_size=1024, + num_experts=8, + micro_batch=1, + moe_router_topk=4, + n_shared_expert=2, + seq_length=2048, + enable_sequence_parallel=True, + kv_lora_rank=512, + num_attention_heads= 128 + 64, + q_lora_rank=16, + qk_nope_dim=128, + qk_rope_dim=64, + tensor_model_parallel_size=1, + v_head_dim=128 + ) + model = DeepSeekMoE(args) + y = model(x) + return y + + +def test_expert( + m: int = 2048 * 32, + n: int = 1024, + k: int = 2048 * 32, +) -> tuple[int, int]: + """Test DeepSeek's expert MLP. + + i.e. self.w2(F.silu(self.w1(x)) * self.w3(x)) + with torch.matmul (BF16) and deep_gemm (FP8). + """ + x = torch.randn(m, k).to(torch.bfloat16) + w = torch.randn(k, n).to(torch.bfloat16) + w = w.t() + w2 = torch.randn(k, n).to(torch.bfloat16) + w2 = w2.t() + o = torch.randn(m, n).to(torch.bfloat16) + o2 = torch.randn(m, n).to(torch.bfloat16) + + w3 = torch.randn(n, k).to(torch.bfloat16) + w3_fp8 = per_block_cast_to_fp8(w3.t()) + + out = torch.randn(m, k).to(torch.bfloat16) + w_fp8 = per_block_cast_to_fp8(w) + w2_fp8 = per_block_cast_to_fp8(w2) + x_fp8 = per_token_cast_to_fp8(x) + + _, t0 = do_deepgemm(x_fp8, w_fp8, w2_fp8, w3_fp8, o, o2, out) + _, t1 = do_matmul(x, w, w3, o, out) + return t0, t1 + + +@cuda_timing_decorator +def do_deepgemm( + x: tuple[torch.Tensor, torch.Tensor], + w: tuple[torch.Tensor, torch.Tensor], + w2: tuple[torch.Tensor, torch.Tensor], + w3: torch.Tensor, + o: torch.Tensor, + o2: torch.Tensor, + out: torch.Tensor, +) -> tuple[int, int]: + deep_gemm.gemm_fp8_fp8_bf16_nt(x, w, o) + deep_gemm.gemm_fp8_fp8_bf16_nt(x, w2, o2) + o2 = F.silu(o) + o2 + o2_fp8 = per_token_cast_to_fp8(o2) + deep_gemm.gemm_fp8_fp8_bf16_nt(o2_fp8, w3, out) + return out + + +@cuda_timing_decorator +def do_matmul( + x: torch.Tensor, + w: torch.Tensor, + w3: torch.Tensor, + o: torch.Tensor, + out: torch.Tensor, +) -> torch.Tensor: + torch.matmul(x, w.t(), out=o) + return torch.matmul(F.silu(o), w3, out=out) + +device = torch.device("cuda:0") +torch.set_default_device(device) + +if __name__ == "__main__": + # torch.set_default_dtype(torch.bfloat16) + device = torch.device("cuda:0") + torch.set_default_device(device) + torch.cuda.set_device(device) + torch.manual_seed(0) + random.seed(0) + + # run once to load/cache kernels + test_deepseek_mlp() + test_expert() + test_deepseek_mla() + + # for kineto traces + import torch.profiler + + with torch.profiler.profile( + on_trace_ready=torch.profiler.tensorboard_trace_handler("./log"), + record_shapes=True, + with_flops=True, + ) as prof: + test_deepseek_mlp() + t0, t1 = test_expert() + print(f"matmul: deepgemm: {t0} matmul: {t1} diff: {t1 - t0}") + + with torch.profiler.profile( + on_trace_ready=torch.profiler.tensorboard_trace_handler("./log"), + record_shapes=True, + with_flops=True, + ) as prof: + _, t = test_deepseek_mla() + print(f"test_deepseek_mla: compute time {t}") diff --git a/aicb/workload_generator/mocked_model/training/AiobMegatron.py b/aicb/workload_generator/mocked_model/training/AiobMegatron.py new file mode 100755 index 000000000..d9fea9475 --- /dev/null +++ b/aicb/workload_generator/mocked_model/training/AiobMegatron.py @@ -0,0 +1,1107 @@ +""" +Copyright (c) 2021, Alibaba Group; +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import torch +import time +import warnings +import torch.nn.functional as F +from apex.contrib.layer_norm.layer_norm import FastLayerNormFN +import math +import scaled_upper_triang_masked_softmax_cuda +from torch.cuda.amp import custom_bwd, custom_fwd +from utils.utils import * +from core import grouped_gemm_util as gg +try: + from einops import rearrange +except ImportError as e: + rearrange = None + print("Failed to import 'einops'. Functions using 'rearrange' might not work.") +from typing import Callable, Optional + +try: + from flash_attn.flash_attn_interface import flash_attn_unpadded_func +except ImportError: + try: + from flash_attn.flash_attn_interface import ( + flash_attn_varlen_func as flash_attn_unpadded_func, + ) + except ImportError: + flash_attn_unpadded_func = None + +from workload_generator.mocked_model.training.AiobDeepSeek import DeepSeekMoE, DeepSeekMLA + +class MegatronModel(torch.nn.Module): + def __init__(self, args=None): + super(MegatronModel, self).__init__() + self.time_list = {} + self.args = args + + self.Embedding = MegatronEmbedding(self.args) + self.Layernorm = MegatronLayernorm(self.args) + if args.frame == "DeepSeek": + self.Attention = DeepSeekMLA(self.args) + if args.frame == "Megatron": + if self.args.use_flash_attn: + self.Attention = MegatronFlashAtten(self.args) + else: + self.Attention = MegatronAtten(self.args) + if self.args.moe_enable: + if self.args.frame == "DeepSeek": + self.Mlp = DeepSeekMoE(self.args) + else: + self.Mlp = MoELayer(self.args) + else: + self.Mlp = MegatronMlp(self.args) + self.logit = logit(self.args) + self.grad_param = Grad_param(self.args) + + def forward(self, input): + for _ in range(self.args.epoch_num): + # #Embedding + Emb_output, Emb_time = self.Embedding(input) + self.time_list.setdefault("Emb", []).append({"time_gpu": Emb_time}) + + for _ in range(self.args.num_layers): + # #layernorm + lay_out, layernorm = self.Layernorm(Emb_output) + self.time_list.setdefault("layernorm", []).append( + {"time_gpu": layernorm} + ) + if self.args.frame == "DeepSeek": + atten_output, time_map = self.Attention(lay_out) + for k, v in time_map.items(): + self.time_list.setdefault(k, []).append( + {"time_gpu": v} + ) + if self.args.frame == "Megatron": + if self.args.use_flash_attn: + atten_output, atten_qkv, atten_core, atten_linear = self.Attention( + lay_out + ) + self.time_list.setdefault("atten_qkv", []).append( + {"time_gpu": atten_qkv} + ) + self.time_list.setdefault("atten_flash", []).append( + {"time_gpu": atten_core} + ) + self.time_list.setdefault("atten_linear", []).append( + {"time_gpu": atten_linear} + ) + else: + ( + atten_output, + atten_qkv, + atten_core_qk, + atten_core_softmax, + atten_core_contex, + atten_linear, + ) = self.Attention(lay_out) + self.time_list.setdefault("atten_qkv", []).append( + {"time_gpu": atten_qkv} + ) + self.time_list.setdefault("atten_core_qk", []).append( + {"time_gpu": atten_core_qk} + ) + self.time_list.setdefault("atten_core_softmax", []).append( + {"time_gpu": atten_core_softmax} + ) + self.time_list.setdefault("atten_core_contex", []).append( + {"time_gpu": atten_core_contex} + ) + self.time_list.setdefault("atten_linear", []).append( + {"time_gpu": atten_linear} + ) + # layernorm + lay2_out, layernorm2 = self.Layernorm(atten_output) + + # mlp layer + mlp_out, mlp_linear_1, mlp_gelu, mlp_linear_2 = self.Mlp(lay2_out) + self.time_list.setdefault("layernorm2", []).append( + {"time_gpu": layernorm2} + ) + self.time_list.setdefault("mlp_linear_1", []).append( + {"time_gpu": mlp_linear_1} + ) + self.time_list.setdefault("mlp_gelu", []).append({"time_gpu": mlp_gelu}) + self.time_list.setdefault("mlp_linear_2", []).append( + {"time_gpu": mlp_linear_2} + ) + + lay_post__out, layernorm_post = self.Layernorm(mlp_out) + self.time_list.setdefault("layernorm_post", []).append( + {"time_gpu": layernorm_post} + ) + print(f"lay_post__out.shape: {lay_post__out.shape}") + logit_out, logit_time = self.logit(lay_post__out) + self.time_list.setdefault("logit_time", []).append({"time_gpu": logit_time}) + _, param_time = self.grad_param._apply() + + self.time_list.setdefault("param_time", []).append({"time_gpu": param_time}) + + filepath = write_op(self.time_list, self.args) + process_all_keys(filepath) + + return filepath + + +class LinearWithGradAccumulationAndAsyncCommunication(torch.autograd.Function): + """See linear_with_grad_accumulation_and_async_allreduce""" + + @staticmethod + @custom_fwd + def forward( + ctx, + input, + weight, + bias, + gradient_accumulation_fusion, + async_grad_allreduce, + sequence_parallel, + tp, + ): + ctx.save_for_backward(input, weight) # + ctx.use_bias = bias is not None + ctx.gradient_accumulation_fusion = gradient_accumulation_fusion + ctx.async_grad_allreduce = async_grad_allreduce + ctx.sequence_parallel = sequence_parallel + + if sequence_parallel: + + total_input = input + else: + total_input = input + + output = torch.matmul(total_input, weight.t()) + + if bias is not None: + output = output + bias + return output + + +def linear_with_grad_accumulation_and_async_allreduce( + input: torch.Tensor, + weight: torch.Tensor, + bias: Optional[torch.Tensor], + gradient_accumulation_fusion: bool, + async_grad_allreduce: bool, + sequence_parallel: bool, + tp, +) -> torch.Tensor: + """Linear layer execution with asynchronous communication and + gradient accumulation fusion in backprop. + + This has the option to accumulate the result of backprop + calculation into an existing gradient buffer, preventing the need + to do an additional addition kernel after the gradient + calculation. + + Additionally, the tensor parallel all reduce of the input + gradients can be done asynchronously with the calculation of + the weight gradients. + + In the case of sequence parallelism, the reduce scatter of the + input gradients is done asynchronously with the calcluation of the + weight gradients. + + Use of this module requires that the environment variable + CUDA_DEVICE_MAX_CONNECTIONS=1. There are a few collective + operations, noted in the code, that should be scheduled before + compute kernels to overlap the communication with the computation, + which is necessary for a speedup but not for correctness so that + ordering isn't imposed by the scheduler. Setting + CUDA_DEVICE_MAX_CONNECTIONS=1 forces the kernels to be scheduled + in the order they are called. + + Arguments: + + input (torch.Tensor required): input like torch.nn.functional.linear + + weight (torch.Tensor required): weight like torch.nn.functional.linear + + bias (torch.Tensor optional): bias like torch.nn.functional.linear + + gradient_accumulation_fusion (bool required): Perform the gradient + accumulation fusion, requires the custom CUDA extension + fused_weight_gradient_mlp_cuda module. To use + gradient_accumulation_fusion you must install APEX with + --cpp_ext and --cuda_ext. For example: "pip install + --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext .\" + " Note that the extension requires CUDA>=11. Otherwise, you + must turn off gradient accumulation fusion." + + async_grad_allreduce (bool required): Do the allreduce of input + gradients asyncronously with the computation of weight + gradients. If sequence_parallel is True, this must be + False, as no all reduce is performed. + + sequence_parallel (bool required): Indicates that sequence + parallelism is used and thus in the forward pass the input is + all gathered, and the backward pass the input gradients are + reduce scattered. + """ + args = [ + input, + weight, + bias, + gradient_accumulation_fusion, + async_grad_allreduce, + sequence_parallel, + tp, + ] + + return LinearWithGradAccumulationAndAsyncCommunication.apply(*args) + + +linear_with_grad_accumulation_and_async_allreduce.warned = False + + +# Embedding +class MegatronEmbedding(torch.nn.Module): + def __init__(self, args=None): + super(MegatronEmbedding, self).__init__() + self.tp = args.tensor_model_parallel_size + micro_batch = args.micro_batch + seq_len = args.seq_length + hidden_size = args.hidden_size + max_position_embeddings = args.max_position_embeddings + self.vocab_size = args.padded_vocab_size + device = torch.cuda.current_device() + if args.dtype == "bfloat16": + self.dtype = torch.bfloat16 + elif args.dtype == "float16": + self.dtype = torch.float16 + else: + self.dtype = torch.float32 + + # self.masked_input = torch.randint(0,math.ceil(self.vocab_size/self.tp), + # (micro_batch,seq_len), + # device=device, dtype=torch.int64) + self.weight = torch.randint( + 0, + 1, + (math.ceil(self.vocab_size / self.tp), hidden_size), + device=device, + dtype=torch.int64, + ) + self.position_embeddings = torch.nn.Embedding( + max_position_embeddings, hidden_size + ).to(device) + self.position_ids = torch.randint( + 0, seq_len - 1, (1, seq_len), device=device, dtype=torch.int64 + ) + + @cuda_timing_decorator + def _apply(self, input): + if self.tp > 1: + # Build the mask. + input_mask = (input < 0) | (input >= math.ceil(self.vocab_size / self.tp)) + # Mask the input. + masked_input = input.clone() - 0 + masked_input[input_mask] = 0 + + else: + masked_input = input + words_embeddings = self.weight[masked_input] + + position_embeddings_i = self.position_embeddings(self.position_ids) + + embeddings = words_embeddings + position_embeddings_i + embeddings = embeddings.transpose(0, 1).contiguous() + + return embeddings + + def forward(self, input): + + result, emb_time = self._apply(input) + + result = result.to(self.dtype) + + return result, emb_time + + +class MegatronLayernorm(torch.nn.Module): + def __init__(self, args=None): + super(MegatronLayernorm, self).__init__() + self.tp = args.tensor_model_parallel_size + self.enable_sequence_parallel = args.enable_sequence_parallel + self.hidden_size = args.hidden_size + device = torch.cuda.current_device() + if args.dtype == "bfloat16": + self.dtype = torch.bfloat16 + elif args.dtype == "float16": + self.dtype = torch.float16 + else: + self.dtype = torch.float32 + # self.input_l = torch.rand(seq_len, + # micro_batch, + # hidden_size, + # device=device).to(dtype) + self.lay_weight = torch.rand(self.hidden_size, device=device).to(self.dtype) + self.bias = torch.zeros(self.hidden_size, device=device).to(self.dtype) + + @cuda_timing_decorator + def _apply_fused_layer_norm(self, hidden_states): + output_lay = FastLayerNormFN.apply( + hidden_states, self.lay_weight, self.bias, 1e-05 + ) + return output_lay + + @cuda_timing_decorator + def _apply_torch_layernorm(self, hidden_states): + output_lay = torch.nn.functional.layer_norm( + hidden_states, [self.hidden_size] , self.lay_weight, self.bias, 1e-05 + ) + return output_lay + + def forward(self, hidden_states): + # hidden_states = hidden_states.to(self.dtype) + if self.enable_sequence_parallel: + chunks = torch.chunk(hidden_states, self.tp, 0) + hidden_states = chunks[0] + + # in case of DeepSeek16B, for Hidden size 10944, FastLayerNormFN fails with + # FWD: Unsupported hidden_size or types: 10944BFloat16BFloat16BFloat16Float + # because https://github.com/NVIDIA/apex/blob/4bdecd06b3c4b2c0a8fb6603829a8f9f05a42b49/apex/contrib/csrc/layer_norm/ln_fwd_cuda_kernel.cu#L73-L227 + # thus, use torch's layer_norm + + # try Fused, if exception, use torch + try: + lay_out, lay_time = self._apply_fused_layer_norm(hidden_states) + except Exception as e: + print(f"FastLayerNormFN failed with error {e}, using torch.nn.functional.layer_norm") + lay_out, lay_time = self._apply_torch_layernorm(hidden_states) + + + if self.enable_sequence_parallel: + lay_out = lay_out.repeat((self.tp, 1, 1)) + + return lay_out, lay_time + + +class MegatronAtten(torch.nn.Module): + def __init__(self, args=None): + super(MegatronAtten, self).__init__() + self.enable_sequence_parallel = args.enable_sequence_parallel + self.tp = args.tensor_model_parallel_size + micro_batch = args.micro_batch + seq_len = args.seq_length + hidden_size = args.hidden_size + num_attention_heads = args.num_attention_heads + self.input_in_float16 = False + if args.dtype == "bfloat16": + dtype = torch.bfloat16 + elif args.dtype == "float16": + dtype = torch.float16 + self.input_in_float16 = True + else: + dtype = torch.float32 + device = torch.cuda.current_device() + # self.atten_total_input_1 = torch.rand(seq_len, + # micro_batch, + # hidden_size, + # device=device).to(dtype) + self.atten_weight_1 = torch.rand( + divide((3 * hidden_size), self.tp), hidden_size, device=device + ).to(dtype) + self.hidden_size_per_partition = divide(hidden_size, self.tp) + self.num_attention_heads_per_partition = divide(num_attention_heads, self.tp) + self.hidden_size_per_attention_head = divide(hidden_size, num_attention_heads) + self.num_query_groups_per_partition = self.num_attention_heads_per_partition + query_layer = torch.rand( + seq_len, + micro_batch, + self.num_attention_heads_per_partition, + self.hidden_size_per_attention_head, + device=device, + ).to(dtype) + key_layer = query_layer + value_layer = query_layer + output_size = ( + query_layer.size(1), + query_layer.size(2), + query_layer.size(0), + key_layer.size(0), + ) + self.query_layer = query_layer.view( + output_size[2], output_size[0] * output_size[1], -1 + ) + self.key_layer = key_layer.view( + output_size[3], output_size[0] * output_size[1], -1 + ) + self.matmul_input_buffer = torch.zeros( + output_size[0] * output_size[1], + output_size[2], + output_size[3], + device=device, + ).to(dtype) + self.scale_t = torch.tensor(1).to(dtype) + soft_input = torch.rand(output_size, device=device).to(dtype) + self.b, self.np, self.sq, self.sk = soft_input.size() + self.soft_input_1 = soft_input.view(-1, self.sq, self.sk) + self.output_size_2 = ( + value_layer.size(1), + value_layer.size(2), + self.query_layer.size(0), + value_layer.size(3), + ) + self.value_layer = value_layer.view( + value_layer.size(0), self.output_size_2[0] * self.output_size_2[1], -1 + ) + self.atten_linear_weight = torch.rand( + hidden_size, self.hidden_size_per_partition, device=device + ).to(dtype) + # self.linear_function = LinearWithGradAccumulationAndAsyncCommunication.apply + def get_batch_per_block(self, sq, sk, b, np): + import scaled_masked_softmax_cuda + + return scaled_masked_softmax_cuda.get_batch_per_block(sq, sk, b, np) + def is_kernel_available(self, b, np, sq, sk): + attn_batches = b * np + + if ( + self.input_in_float16 # input must be fp16 + and 16 < sk <= 16384 # sk must be 16 ~ 16384 + and sq % 4 == 0 # sq must be divisor of 4 + and sk % 4 == 0 # sk must be divisor of 4 + and attn_batches % 4 == 0 # np * b must be divisor of 4 + ): + if 0 <= sk <= 16384: + batch_per_block = self.get_batch_per_block(sq, sk, b, np) + + if attn_batches % batch_per_block == 0: + return True + + return False + @cuda_timing_decorator + def _apply_attenqkv(self, hideen_states): + _forward_impl = linear_with_grad_accumulation_and_async_allreduce + output = _forward_impl( + input=hideen_states, + weight=self.atten_weight_1, + bias=None, + gradient_accumulation_fusion=True, + async_grad_allreduce=False, + sequence_parallel=self.enable_sequence_parallel, + tp=self.tp, + ) + # linear_function = LinearWithGradAccumulationAndAsyncCommunication.apply + # output = linear_function(self.atten_total_input_1,self.atten_weight_1,None,None,False,False) + # # output = torch.matmul(self.atten_total_input_1, self.atten_weight_1) + new_tensor_shape = output.size()[:-1] + ( + self.num_query_groups_per_partition, + ( + ( + self.num_attention_heads_per_partition + // self.num_query_groups_per_partition + + 2 + ) + * self.hidden_size_per_attention_head + ), + ) + output = output.view(*new_tensor_shape) + (query_layer, key_layer, value_layer) = torch.split( + output, + [ + ( + self.num_attention_heads_per_partition + // self.num_query_groups_per_partition + * self.hidden_size_per_attention_head + ), + self.hidden_size_per_attention_head, + self.hidden_size_per_attention_head, + ], + dim=3, + ) + return query_layer, key_layer, value_layer + + @cuda_timing_decorator + def _apply_QK(self, q, k): + matmul_result = torch.baddbmm( + self.matmul_input_buffer, + q.transpose(0, 1), # [b * np, sq, hn] + k.transpose(0, 1).transpose(1, 2), # [b * np, hn, sk] + beta=0.0, + alpha=(1.0 / 11.313708498984761), + ) + return matmul_result + + @cuda_timing_decorator + def _apply_Softmax(self,attention_scores): + if self.is_kernel_available(*attention_scores.size()): + b, np, sq, sk = attention_scores.size() + attention_scores = attention_scores.view(-1, sq, sk) + softmax_results = scaled_upper_triang_masked_softmax_cuda.forward( + attention_scores, self.scale_t + ) + prob = softmax_results.view(self.b, self.np, self.sq, self.sk) + else: + if self.scale_t is not None: + attention_scores = attention_scores * self.scale_t + prob = torch.nn.Softmax(dim=-1)(attention_scores) + + return prob + + @cuda_timing_decorator + def _apply_Contex(self, prob, value_layer): + value_layer = value_layer.view( + value_layer.size(0), self.output_size_2[0] * self.output_size_2[1], -1 + ) + attention_probs = prob.view( + self.output_size_2[0] * self.output_size_2[1], self.output_size_2[2], -1 + ) + output = torch.bmm(attention_probs, value_layer.transpose(0, 1)) + context_layer = ( + output.view(*self.output_size_2).permute(2, 0, 1, 3).contiguous() + ) + new_context_layer_shape = context_layer.size()[:-2] + ( + self.hidden_size_per_partition, + ) + context_layer = context_layer.view(*new_context_layer_shape) + return context_layer + + @cuda_timing_decorator + def _apply_Linear(self, context_layer): + _forward_impl = linear_with_grad_accumulation_and_async_allreduce + output_parallel = _forward_impl( + input=context_layer, + weight=self.atten_linear_weight, + bias=None, + gradient_accumulation_fusion=True, + async_grad_allreduce=False, + sequence_parallel=self.enable_sequence_parallel, + tp=self.tp, + ) + return output_parallel + + def forward(self, hideen_states): + qkv_out, qkv_time = self._apply_attenqkv(hideen_states) + + query_layer, key_layer, value_layer = qkv_out + + output_size = ( + query_layer.size(1), + query_layer.size(2), + query_layer.size(0), + key_layer.size(0), + ) + query_layer = query_layer.reshape( + output_size[2], output_size[0] * output_size[1], -1 + ) + key_layer = key_layer.view(output_size[3], output_size[0] * output_size[1], -1) + matmul_result, qk_time = self._apply_QK(query_layer, key_layer) + attention_scores = matmul_result.view(*output_size) + softmax_results, softmax_time = self._apply_Softmax(attention_scores) + context_layer, contex_time = self._apply_Contex(softmax_results, value_layer) + output, attrn_linear_time = self._apply_Linear(context_layer) + return output, qkv_time, qk_time, softmax_time, contex_time, attrn_linear_time + + +class MegatronFlashAtten(torch.nn.Module): + def __init__(self, args=None): + super(MegatronFlashAtten, self).__init__() + self.enable_sequence_parallel = args.enable_sequence_parallel + self.tp = args.tensor_model_parallel_size + micro_batch = args.micro_batch + seq_len = args.seq_length + + hidden_size = args.hidden_size + num_attention_heads = args.num_attention_heads + if args.dtype == "bfloat16": + dtype = torch.bfloat16 + elif args.dtype == "float16": + dtype = torch.float16 + else: + dtype = torch.float32 + device = torch.cuda.current_device() + + self.atten_weight_1 = torch.rand( + divide((3 * hidden_size), self.tp), hidden_size, device=device + ).to(dtype) + + self.hidden_size_per_partition = divide(hidden_size, self.tp) + self.num_attention_heads_per_partition = divide(num_attention_heads, self.tp) + self.hidden_size_per_attention_head = divide(hidden_size, num_attention_heads) + self.num_query_groups_per_partition = self.num_attention_heads_per_partition + + self.atten_linear_weight = torch.rand( + hidden_size, self.hidden_size_per_partition, device=device + ).to(dtype) + + @cuda_timing_decorator + def _apply_attenqkv(self, hideen_states): + _forward_impl = linear_with_grad_accumulation_and_async_allreduce + + output = _forward_impl( + input=hideen_states, + weight=self.atten_weight_1, + bias=None, + gradient_accumulation_fusion=True, + async_grad_allreduce=False, + sequence_parallel=self.enable_sequence_parallel, + tp=self.tp, + ) + + new_tensor_shape = output.size()[:-1] + ( + self.num_query_groups_per_partition, + ( + ( + self.num_attention_heads_per_partition + // self.num_query_groups_per_partition + + 2 + ) + * self.hidden_size_per_attention_head + ), + ) + output = output.view(*new_tensor_shape) + (query_layer, key_layer, value_layer) = torch.split( + output, + [ + ( + self.num_attention_heads_per_partition + // self.num_query_groups_per_partition + * self.hidden_size_per_attention_head + ), + self.hidden_size_per_attention_head, + self.hidden_size_per_attention_head, + ], + dim=3, + ) + + return query_layer, key_layer, value_layer + + @cuda_timing_decorator + def _apply_flash_atten(self, q, k, v): + + output = flash_attn_unpadded_func( + q, + k, + v, + self.cu_seqlens_q, + self.cu_seqlens_k, + self.seqlen_q, + self.seqlen_k, + 0.0, + softmax_scale=None, + causal=True, + ) + + context_layer = rearrange(output, "(b s) ... -> b s ...", b=self.micro_batch) + + return context_layer + + @cuda_timing_decorator + def _apply_Linear(self, context_layer): + _forward_impl = linear_with_grad_accumulation_and_async_allreduce + context_layer = rearrange(context_layer, "b s h d -> s b (h d)").contiguous() + output_parallel = _forward_impl( + input=context_layer, + weight=self.atten_linear_weight, + bias=None, + gradient_accumulation_fusion=True, + async_grad_allreduce=False, + sequence_parallel=self.enable_sequence_parallel, + tp=self.tp, + ) + return output_parallel + + def forward(self, hidden_state): + if rearrange is None: + raise ImportError( + "The function 'rearrange' from 'einops' is required but not available." + ) + result, qkv_time = self._apply_attenqkv(hidden_state) + q, k, v = result + q, k, v = [rearrange(x, "s b ... -> b s ...").contiguous() for x in (q, k, v)] + self.micro_batch, self.seqlen_q = q.shape[0], q.shape[1] + self.seqlen_k = k.shape[1] + + q, k, v = [rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v]] + self.cu_seqlens_q = torch.arange( + 0, + (self.micro_batch + 1) * self.seqlen_q, + step=self.seqlen_q, + dtype=torch.int32, + device=q.device, + ) + + if self.training: + assert self.seqlen_k == self.seqlen_q + self.cu_seqlens_k = self.cu_seqlens_q + context_layer, flash_time = self._apply_flash_atten(q, k, v) + output, attrn_linear_time = self._apply_Linear(context_layer) + return output, qkv_time, flash_time, attrn_linear_time + + +class MegatronMlp(torch.nn.Module): + def __init__(self, args=None): + super(MegatronMlp, self).__init__() + self.tp = args.tensor_model_parallel_size + self.enable_sequence_parallel = args.enable_sequence_parallel + micro_batch = args.micro_batch + seq_len = args.seq_length + self.add_bias_linear = False + if args.add_bias_linear: + self.add_bias_linear = True + + hidden_size = args.hidden_size + ffn_hidden_size = args.ffn_hidden_size + if args.gated_linear_unit: + ffn_hidden_size *= 2 + num_attention_heads = args.num_attention_heads + device = torch.cuda.current_device() + if args.dtype == "bfloat16": + dtype = torch.bfloat16 + elif args.dtype == "float16": + dtype = torch.float16 + else: + dtype = torch.float32 + # activation + if args.openai_gelu: + self.activation_func = openai_gelu + elif args.onnx_safe: + self.activation_func = erf_gelu + elif args.swiglu: + + def swiglu(x): + x = torch.chunk(x, 2, dim=-1) + + return F.silu(x[0]) * x[1] + + self.activation_func = swiglu + elif args.squared_relu: + + def squared_relu(x): + return torch.pow(F.relu(x), 2) + + self.activation_func = squared_relu + else: + self.bias_gelu_fusion = args.bias_gelu_fusion + self.activation_func = F.gelu + output_size_per_partition = divide(ffn_hidden_size, self.tp) + + self.weight_1 = torch.rand( + output_size_per_partition, hidden_size, device=device + ).to(dtype) + self.bias = torch.empty(output_size_per_partition, device=device).to(dtype) + + self.weight_2 = torch.rand( + hidden_size, args.ffn_hidden_size // self.tp, device=device + ).to(dtype) + + @cuda_timing_decorator + def _apply_Linear1(self, hidden_state): + _forward_impl = linear_with_grad_accumulation_and_async_allreduce + output_parallel = _forward_impl( + input=hidden_state, + weight=self.weight_1, + bias=None, + gradient_accumulation_fusion=True, + async_grad_allreduce=False, + sequence_parallel=self.enable_sequence_parallel, + tp=self.tp, + ) + return output_parallel + + @cuda_timing_decorator + def _apply_activation(self, hidden_state): + if self.add_bias_linear: + intermediate_parallel = self.activation_func(hidden_state + self.bias) + else: + intermediate_parallel = self.activation_func(hidden_state) + + return intermediate_parallel + + @cuda_timing_decorator + def _apply_Linear2(self, hidden_state): + _forward_impl = linear_with_grad_accumulation_and_async_allreduce + output_parallel = _forward_impl( + input=hidden_state, + weight=self.weight_2, + bias=None, + gradient_accumulation_fusion=True, + async_grad_allreduce=False, + sequence_parallel=self.enable_sequence_parallel, + tp=self.tp, + ) + return output_parallel + + def forward(self, hidden_state): + l1_out, l1_time = self._apply_Linear1(hidden_state) + act_out, act_time = self._apply_activation(l1_out) + l2_out, l2_time = self._apply_Linear2(act_out) + return l2_out, l1_time, act_time, l2_time + + +class logit(torch.nn.Module): + def __init__(self, args=None): + super(logit, self).__init__() + self.enable_sequence_parallel = args.enable_sequence_parallel + self.tp = args.tensor_model_parallel_size + micro_batch = args.micro_batch + seq_len = args.seq_length + vocab_size = args.padded_vocab_size + hidden_size = args.hidden_size + ffn_hidden_size = args.ffn_hidden_size + if args.gated_linear_unit: + ffn_hidden_size *= 2 + num_attention_heads = args.num_attention_heads + device = torch.cuda.current_device() + if args.dtype == "bfloat16": + dtype = torch.bfloat16 + elif args.dtype == "float16": + dtype = torch.float16 + else: + dtype = torch.float32 + output_size_per_partition = divide(vocab_size, self.tp) + self.word_embeddings_weight = torch.rand( + output_size_per_partition, hidden_size, device=device, requires_grad=True + ).to(dtype) + + @cuda_timing_decorator + def _apply(self, hidden_state): + _forward_impl = linear_with_grad_accumulation_and_async_allreduce + output_parallel = _forward_impl( + input=hidden_state, + weight=self.word_embeddings_weight, + bias=None, + gradient_accumulation_fusion=True, + async_grad_allreduce=True, + sequence_parallel=self.enable_sequence_parallel, + tp=self.tp, + ) + return output_parallel + + def forward(self, input): + log_out, log_time = self._apply(input) + return log_out, log_time + + +class Grad_param: + def __init__(self, args=None): + tp = args.tensor_model_parallel_size + param = args.model_param + self.dp = args.dp_num + + device = torch.cuda.current_device() + dtype = torch.float32 + self.data = torch.rand(param//tp, device=device).to(dtype) + + @cuda_timing_decorator + def _apply(self): + + self.data /= self.dp + # assert self.data.numel() % self.dp == 0 + shard_size = self.data.numel() // self.dp + sharded_buffer = [ + self.data[(r * shard_size) : ((r + 1) * shard_size)] for r in range(self.dp) + ] + return sharded_buffer +class SequentialMLP(torch.nn.Module): + """An implementation of the Experts layer using a sequence of MLP layers. + + This class executes each expert sequentially. + """ + + def __init__(self, num_local_experts,args=None): + super(SequentialMLP,self).__init__() + tp = args.tensor_model_parallel_size + ep = args.expert_model_parallel_size + num_experts = args.num_experts + micro_batch = args.micro_batch + seq_len = args.seq_length + topk = args.moe_router_topk + hidden_size = args.hidden_size + num_attention_heads = args.num_attention_heads + self.add_bias = False + # self.moe_extended_tp = config.moe_extended_tp + self.num_local_experts = num_local_experts + self.local_experts = torch.nn.ModuleList() + for _ in range(self.num_local_experts): + expert = MegatronMlp(args) + self.local_experts.append(expert) + + def forward(self, permuted_local_hidden_states, tokens_per_expert): + + output_local = torch.zeros_like(permuted_local_hidden_states) + output_bias_local = None + if self.add_bias: + output_bias_local = torch.zeros_like(permuted_local_hidden_states) + + cumsum_num_tokens = torch.cumsum(tokens_per_expert, dim=0) + # Insert zero at the begining for offset index's convenience + zero_tensor = torch.zeros(1, dtype=torch.long, device=cumsum_num_tokens.device) + cumsum_num_tokens = torch.cat((zero_tensor, cumsum_num_tokens)) + mlp_linear_1_all , mlp_gelu_all ,mlp_linear_2_all = 0 ,0 ,0 + for expert_num, expert in enumerate(self.local_experts): + start = cumsum_num_tokens[expert_num] + end = cumsum_num_tokens[expert_num + 1] + + hidden = permuted_local_hidden_states[start:end] + # output, output_bias = expert(hidden) + output,mlp_linear_1,mlp_gelu,mlp_linear_2 = expert(hidden) + + output_local[start:end] = output + mlp_linear_1_all += mlp_linear_1 + mlp_gelu_all += mlp_gelu + mlp_linear_2_all += mlp_linear_2 + if self.add_bias: + output_bias = output_bias.expand_as(output) + output_bias_local[start:end, :] = output_bias + + return output_local, mlp_linear_1_all,mlp_gelu_all,mlp_linear_2_all +class GroupedMLP(torch.nn.Module): + """An efficient implementation of the Experts layer using CUTLASS GroupedGEMM. + + This class is designed to execute multiple experts in parallel, thereby maximizing computational efficiency. + """ + + def __init__(self, num_local_experts,args=None): + super(GroupedMLP,self).__init__() + self.num_local_experts = num_local_experts + gg.assert_grouped_gemm_is_available() + tp = args.tensor_model_parallel_size + self.hidden_size = args.hidden_size + self.expert_parallel = args.expert_model_parallel_size > 1 + device = torch.cuda.current_device() + if args.dtype == "bfloat16": + dtype = torch.bfloat16 + elif args.dtype == "float16": + dtype = torch.float16 + else: + dtype = torch.float32 + if args.openai_gelu: + self.activation_func = openai_gelu + elif args.onnx_safe: + self.activation_func = erf_gelu + elif args.swiglu: + def swiglu(x): + x = torch.chunk(x, 2, dim=-1) + + return F.silu(x[0]) * x[1] + self.activation_func = swiglu + elif args.squared_relu: + def squared_relu(x): + return torch.pow(F.relu(x), 2) + self.activation_func = squared_relu + else: + self.bias_gelu_fusion = args.bias_gelu_fusion + self.activation_func = F.gelu + # if args.gated_linear_unit: + # def glu(x): + # x = torch.chunk(x, 2, dim=-1) + # return self.config.activation_func(x[0]) * x[1] + + # self.activation_func = glu + # else: + # self.activation_func = self.activation_func + + # How many feature each rank holds for fc1 and fc2, respectively. + # self.moe_extended_tp = config.moe_extended_tp + # if config.moe_extended_tp: + # tp_size = parallel_state.get_tensor_and_expert_parallel_world_size() + # else: + # tp_size = parallel_state.get_tensor_model_parallel_world_size() + + fc1_output_size = args.ffn_hidden_size * self.num_local_experts + if args.gated_linear_unit: + # Project to 4h. If using swiglu double the output width, + # see https://arxiv.org/pdf/2002.05202.pdf + fc1_output_size *= 2 + fc1_output_size_per_partition = divide(fc1_output_size, tp) + + fc2_input_size = args.ffn_hidden_size * self.num_local_experts + fc2_input_size_per_partition = divide(fc2_input_size, tp) + + # Note: The current kernel implementations of grouped_gemm + # does not support transposition with CUTLASS grouped GEMM + # (https://github.com/fanshiqing/grouped_gemm/blob/main/csrc/grouped_gemm.cu#L355-L358) + # and as a result we avoid allocate the transpose of weights. + # Initialize weight. + self.weight1 = torch.rand(self.hidden_size , + fc1_output_size_per_partition, + device=device).to(dtype) + self.weight2 = torch.rand(fc2_input_size_per_partition, + self.hidden_size , + device=device).to(dtype) + @cuda_timing_decorator + def _apply_Linear1(self,permuted_local_hidden_states,tokens_per_expert,w1): + + + + fc1_output = gg.ops.gmm( + permuted_local_hidden_states, w1, tokens_per_expert, trans_b=False + ) + return fc1_output + + @cuda_timing_decorator + def _apply_activation(self,fc1_output): + + intermediate_parallel = self.activation_func(fc1_output) + + return intermediate_parallel + + @cuda_timing_decorator + def _apply_Linear2(self,intermediate_parallel,tokens_per_expert,w2): + + + fc2_output = gg.ops.gmm(intermediate_parallel, w2, tokens_per_expert, trans_b=False) + + return fc2_output + def forward(self, permuted_local_hidden_states, tokens_per_expert): + w1 = self.weight1.view(self.num_local_experts, self.hidden_size, -1) + w2 = self.weight2.view(self.num_local_experts, -1, self.hidden_size ) + l1_out,l1_time = self._apply_Linear1(permuted_local_hidden_states,tokens_per_expert,w1) + act_out,act_time = self._apply_activation(l1_out) + l2_out,l2_time = self._apply_Linear2(act_out,tokens_per_expert,w2) + + return l2_out,l1_time,act_time,l2_time +class MoELayer(torch.nn.Module): + def __init__(self,args=None): + super(MoELayer,self).__init__() + + ep = args.expert_model_parallel_size + num_experts = args.num_experts + micro_batch = args.micro_batch + seq_len = args.seq_length + topk = args.moe_router_topk + hidden_size = args.hidden_size + num_attention_heads = args.num_attention_heads + self.num_local_experts = int(num_experts / ep) + if args.dtype == "bfloat16": + dtype = torch.bfloat16 + elif args.dtype == "float16": + dtype = torch.float16 + else: + dtype = torch.float32 + device = torch.cuda.current_device() + if args.moe_grouped_gemm: + self.experts = GroupedMLP(self.num_local_experts, args) + else: + self.experts = SequentialMLP(self.num_local_experts, args) + # print("aa",seq_len*micro_batch*topk*dp/num_experts*self.num_local_experts) + self.dispatched_input = torch.rand(int(seq_len*micro_batch*topk*ep/num_experts*self.num_local_experts), hidden_size + ,device = device).to(dtype) + temp_val = int(seq_len*micro_batch*topk*ep/num_experts) + # self.tokens_per_expert = torch.tensor([temp,temp],device = device) + + self.tokens_per_expert = torch.full((self.num_local_experts,), temp_val) + # print('aa',self.tokens_per_expert) + def forward(self, hidden_states: torch.Tensor): + # probs, indices = self.router(hidden_states) + # (dispatched_input, tokens_per_expert) = self.token_dispatcher.token_permutation( + # hidden_states, probs, indices + # ) + + expert_output, mlp_linear_1,mlp_gelu,mlp_linear_2 = self.experts(self.dispatched_input, self.tokens_per_expert) + # output, mlp_bias = self.token_dispatcher.token_unpermutation(expert_output, mlp_bias) + return expert_output,mlp_linear_1,mlp_gelu,mlp_linear_2 diff --git a/aicb/workload_generator/mocked_model/training/AiobQwen3.py b/aicb/workload_generator/mocked_model/training/AiobQwen3.py new file mode 100644 index 000000000..a90c166db --- /dev/null +++ b/aicb/workload_generator/mocked_model/training/AiobQwen3.py @@ -0,0 +1,338 @@ +""" +AiobQwen3 -- GPU compute-time benchmark for Qwen3 training workloads. + +Adapted from AiobMegatron.py. Qwen3-specific changes: + - GQA: separate num_key_value_heads with explicit head_dim (128) + - QK-Norm: per-head RMSNorm on Q and K after projection, before attention + - QKV projection size: (n_heads + 2*n_kv_heads) * head_dim instead of 3*hidden + +Requires CUDA GPU with flash-attn, apex, and scaled_upper_triang_masked_softmax. +Not runnable on CPU/MPS -- this file documents the benchmark structure for +machines with the required GPU libraries. +""" + +import torch +import torch.nn.functional as F +import time +import math +import warnings +from utils.utils import divide + +# Optional GPU-only imports -- will fail on CPU/MPS +try: + from apex.contrib.layer_norm.layer_norm import FastLayerNormFN +except ImportError: + FastLayerNormFN = None +try: + from flash_attn.flash_attn_interface import flash_attn_unpadded_func +except ImportError: + try: + from flash_attn.flash_attn_interface import ( + flash_attn_varlen_func as flash_attn_unpadded_func, + ) + except ImportError: + flash_attn_unpadded_func = None + + +# =========================================================================== +# Qwen3-specific GPU kernels +# =========================================================================== + +class Qwen3RMSNorm(torch.nn.Module): + """RMSNorm for QK-Norm (per-head, on head_dim=128).""" + + def __init__(self, dim, eps=1e-6): + super().__init__() + self.weight = torch.nn.Parameter(torch.ones(dim)) + self.eps = eps + + def forward(self, x): + # x: (seq, batch, heads, head_dim) + rms = torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + return x * rms * self.weight + + +class Qwen3Layernorm(torch.nn.Module): + """Standard RMSNorm + QK-Norm timing harness for Qwen3.""" + + def __init__(self, args): + super().__init__() + self.tp = args.tensor_model_parallel_size + self.enable_sequence_parallel = args.enable_sequence_parallel + self.hidden_size = args.hidden_size + seq_len = args.seq_length + micro_batch = args.micro_batch + device = torch.cuda.current_device() + if args.dtype == "bfloat16": + dtype = torch.bfloat16 + elif args.dtype == "float16": + dtype = torch.float16 + else: + dtype = torch.float32 + self.dtype = dtype + + # Standard layernorm weights + self.ln_weight = torch.rand(self.hidden_size, device=device).to(dtype) + self.ln_bias = torch.zeros(self.hidden_size, device=device).to(dtype) + + # QK-Norm weights (2 per layer: q_norm, k_norm) + head_dim = getattr(args, "head_dim", 128) + n_kv_heads = getattr(args, "num_key_value_heads", args.num_attention_heads) + self.q_norm = Qwen3RMSNorm(head_dim, eps=args.rms_norm_eps if hasattr(args, "rms_norm_eps") else 1e-6) + self.k_norm = Qwen3RMSNorm(head_dim, eps=args.rms_norm_eps if hasattr(args, "rms_norm_eps") else 1e-6) + + self.layernorm_input = torch.rand( + seq_len * self.tp if self.enable_sequence_parallel else seq_len, + micro_batch, self.hidden_size, device=device, + ).to(dtype) + + def _apply_fused_layer_norm(self, hidden_states): + torch.cuda.synchronize() + t_start = time.time() + ln_out = FastLayerNormFN.apply( + hidden_states, torch.tensor([self.hidden_size], device=hidden_states.device), + self.ln_weight, self.ln_bias, 1e-5, + ) + torch.cuda.synchronize() + return ln_out, (time.time() - t_start) * 1e6 + + def forward(self, hidden_states): + # Standard layernorm + try: + lay_out, lay_time = self._apply_fused_layer_norm(hidden_states) + except Exception: + torch.cuda.synchronize() + t_start = time.time() + lay_out = F.layer_norm( + hidden_states, [self.hidden_size], self.ln_weight, self.ln_bias, 1e-5, + ) + torch.cuda.synchronize() + lay_time = (time.time() - t_start) * 1e6 + + if self.enable_sequence_parallel: + lay_out = lay_out.repeat((self.tp, 1, 1)) + return lay_out, lay_time + + def qk_norm_forward(self, q, k): + """Time the QK-Norm operations (computed per-head).""" + torch.cuda.synchronize() + t_start = time.time() + q = self.q_norm(q) + k = self.k_norm(k) + torch.cuda.synchronize() + return q, k, (time.time() - t_start) * 1e6 + + +class Qwen3Attention(torch.nn.Module): + """Qwen3 GQA attention with QK-Norm (GPU benchmark harness).""" + + def __init__(self, args): + super().__init__() + self.tp = args.tensor_model_parallel_size + self.enable_sequence_parallel = args.enable_sequence_parallel + hidden_size = args.hidden_size + n_heads = args.num_attention_heads + n_kv = getattr(args, "num_key_value_heads", n_heads) + head_dim = getattr(args, "head_dim", hidden_size // n_heads) + micro_batch = args.micro_batch + seq_len = args.seq_length + + if args.dtype == "bfloat16": + dtype = torch.bfloat16 + elif args.dtype == "float16": + dtype = torch.float16 + else: + dtype = torch.float32 + device = torch.cuda.current_device() + + # GQA-aware fused QKV weight + q_dim = n_heads * head_dim + k_dim = n_kv * head_dim + v_dim = n_kv * head_dim + total_qkv = q_dim + k_dim + v_dim + self.atten_weight_qkv = torch.rand( + divide(total_qkv, self.tp), hidden_size, device=device, + ).to(dtype) + + # Output projection weight + self.hidden_size_per_partition = divide(hidden_size, self.tp) + self.atten_weight_out = torch.rand( + hidden_size, divide(q_dim, self.tp), device=device, + ).to(dtype) + + # Attention core tensors + self.num_heads_per_tp = divide(n_heads, self.tp) + self.num_kv_per_tp = divide(n_kv, self.tp) + self.head_dim = head_dim + + query_layer = torch.rand( + seq_len, micro_batch, self.num_heads_per_tp, head_dim, device=device, + ).to(dtype) + key_layer = torch.rand( + seq_len, micro_batch, self.num_kv_per_tp, head_dim, device=device, + ).to(dtype) + value_layer = torch.rand( + seq_len, micro_batch, self.num_kv_per_tp, head_dim, device=device, + ).to(dtype) + + output_size = ( + query_layer.size(1), query_layer.size(2), + query_layer.size(0), key_layer.size(0), + ) + self.query_layer = query_layer.view(output_size[2], output_size[0] * output_size[1], -1) + self.key_layer = key_layer.view(output_size[3], output_size[0] * output_size[1], -1) + self.value_layer = value_layer.view(output_size[3], output_size[0] * output_size[1], -1) + + self.matmul_input_buffer = torch.zeros( + output_size[0] * output_size[1], output_size[2], output_size[3], + device=device, + ).to(dtype) + self.scale_t = torch.tensor(1).to(dtype) + soft_input = torch.rand(output_size, device=device).to(dtype) + self.b, self.np, self.sq, self.sk = soft_input.size() + + # QK-Norm harness + n_kv_tp = divide(n_kv, self.tp) + self.q_for_norm = torch.rand(seq_len, micro_batch, self.num_heads_per_tp, head_dim, device=device).to(dtype) + self.k_for_norm = torch.rand(seq_len, micro_batch, n_kv_tp, head_dim, device=device).to(dtype) + + def forward(self, hidden_states): + torch.cuda.synchronize() + t_start = time.time() + + # QKV fused projection + mixed_qkv = torch.matmul(hidden_states, self.atten_weight_qkv.t()) + # Split into Q, K, V + q_dim = self.num_heads_per_tp * self.head_dim + k_dim = self.num_kv_per_tp * self.head_dim + q_raw = mixed_qkv[:, :, :q_dim] + k_raw = mixed_qkv[:, :, q_dim:q_dim + k_dim] + v_raw = mixed_qkv[:, :, q_dim + k_dim:] + + qkv_time = (time.time() - t_start) * 1e6 + + return q_raw, k_raw, v_raw, qkv_time + + +class Qwen3Mlp(torch.nn.Module): + """Qwen3 SwiGLU MLP (GPU benchmark harness).""" + + def __init__(self, args): + super().__init__() + self.tp = args.tensor_model_parallel_size + hidden_size = args.hidden_size + intermediate_size = getattr(args, "intermediate_size", args.ffn_hidden_size) + swiglu = getattr(args, "swiglu", False) + if args.dtype == "bfloat16": + dtype = torch.bfloat16 + elif args.dtype == "float16": + dtype = torch.float16 + else: + dtype = torch.float32 + device = torch.cuda.current_device() + + gate_up_dim = 2 * intermediate_size if swiglu else intermediate_size + self.mlp_weight_gate_up = torch.rand( + divide(gate_up_dim, self.tp), hidden_size, device=device, + ).to(dtype) + self.mlp_weight_down = torch.rand( + hidden_size, divide(intermediate_size, self.tp), device=device, + ).to(dtype) + + def forward(self, hidden_states): + torch.cuda.synchronize() + t_start = time.time() + gate_up = torch.matmul(hidden_states, self.mlp_weight_gate_up.t()) + # SiLU activation (swish) + gate_up = F.silu(gate_up) + gate_time = (time.time() - t_start) * 1e6 + + torch.cuda.synchronize() + t_start = time.time() + down = torch.matmul(gate_up, self.mlp_weight_down.t()) + down_time = (time.time() - t_start) * 1e6 + + return gate_time + down_time + + +# =========================================================================== +# Full Qwen3 benchmark model +# =========================================================================== + +class Qwen3Model(torch.nn.Module): + """Complete Qwen3 training benchmark. Same structure as AiobMegatron.MegatronModel.""" + + def __init__(self, args): + super().__init__() + self.time_list = {} + self.args = args + + from workload_generator.mocked_model.training.AiobMegatron import ( + MegatronEmbedding, MegatronLayernorm, logit, Grad_param, + ) + self.Embedding = MegatronEmbedding(self.args) + self.Layernorm = MegatronLayernorm(self.args) + self.Qwen3Layernorm = Qwen3Layernorm(self.args) + self.Attention = Qwen3Attention(self.args) + self.Mlp = Qwen3Mlp(self.args) + self.logit = logit(self.args) + self.grad_param = Grad_param(self.args) + + def forward(self, input): + for _ in range(self.args.epoch_num): + Emb_output, Emb_time = self.Embedding(input) + self.time_list.setdefault("Emb", []).append({"time_gpu": Emb_time}) + + for _ in range(self.args.num_layers): + # Layernorm + lay_out, lay_time = self.Layernorm(Emb_output) + self.time_list.setdefault("layernorm", []).append({"time_gpu": lay_time}) + + # Attention QKV projection + q_raw, k_raw, v_raw, atten_qkv_time = self.Attention(lay_out) + self.time_list.setdefault("atten_qkv", []).append({"time_gpu": atten_qkv_time}) + + # QK-Norm (Qwen3-specific) + q_normed, k_normed, qk_norm_time = self.Qwen3Layernorm.qk_norm_forward(q_raw, k_raw) + self.time_list.setdefault("qk_norm", []).append({"time_gpu": qk_norm_time}) + + # Attention core (Q*K^T, softmax, context) + # Reuse Megatron's attention core timing -- dimensions match after GQA repeat + # TODO: implement full attention core benchmark for GQA + self.time_list.setdefault("atten_core", []).append({"time_gpu": 1}) + + # Attention output projection + atten_out_time = self._time_matmul( + lay_out[:, :, :self.Attention.hidden_size_per_partition], + self.Attention.atten_weight_out, + ) + self.time_list.setdefault("atten_linear", []).append({"time_gpu": atten_out_time}) + + # Post-attention layernorm + lay_out2, lay_time2 = self.Layernorm(lay_out) + self.time_list.setdefault("layernorm2", []).append({"time_gpu": lay_time2}) + + # MLP + mlp_time = self.Mlp(lay_out2) + self.time_list.setdefault("mlp", []).append({"time_gpu": mlp_time}) + + # Final layernorm + logit + grad_param + lay_out_post, post_time = self.Layernorm(Emb_output) + self.time_list.setdefault("layernorm_post", []).append({"time_gpu": post_time}) + + logit_time = self.logit(lay_out_post) + self.time_list.setdefault("logit_time", []).append({"time_gpu": logit_time}) + + grad_time = self.grad_param(lay_out_post) + self.time_list.setdefault("param_time", []).append({"time_gpu": grad_time}) + + from utils.utils import write_time + return write_time(self.time_list, self.args) + + @staticmethod + def _time_matmul(a, b): + torch.cuda.synchronize() + t_start = time.time() + torch.matmul(a, b.t()) + torch.cuda.synchronize() + return (time.time() - t_start) * 1e6 diff --git a/aicb/workload_generator/mocked_model/training/AiobQwen3_5.py b/aicb/workload_generator/mocked_model/training/AiobQwen3_5.py new file mode 100644 index 000000000..e63cc66fc --- /dev/null +++ b/aicb/workload_generator/mocked_model/training/AiobQwen3_5.py @@ -0,0 +1,409 @@ +""" +AiobQwen3_5 -- GPU compute-time benchmark for Qwen3.5 training workloads. + +Adapted from AiobMegatron.py. Qwen3.5-specific changes: + - GatedDeltaNet: 75% of layers use linear attention (Conv1D + Gated Delta Rule) + No softmax, no RoPE. Requires flash-linear-attention (FLA) and causal-conv1d. + - Gated Full Attention (25%): head_dim=256, partial_rotary=0.25, doubled q_proj + with sigmoid gate on output. QK-Norm per-head. + - 3D M-RoPE with mrope_section=[11,11,10] for multimodal position encoding. + - RMSNormGated for DeltaNet output: norm(output) * silu(gate). + +GPU library requirements (NOT available on CPU/MPS): + - flash-linear-attention (chunk_gated_delta_rule, fused_recurrent_gated_delta_rule) + - causal-conv1d (causal_conv1d_fn, causal_conv1d_update) + - flash-attn (flash_attn_unpadded_func) + - apex (FastLayerNormFN) + - scaled_upper_triang_masked_softmax_cuda + +This file documents the benchmark structure. Actual GPU timings require +a CUDA-capable machine with the above libraries installed. +""" + +import torch +import torch.nn.functional as F +import time +import math +import warnings +from utils.utils import divide + +# Optional GPU-only imports +try: + from apex.contrib.layer_norm.layer_norm import FastLayerNormFN +except ImportError: + FastLayerNormFN = None +try: + from flash_attn.flash_attn_interface import flash_attn_unpadded_func +except ImportError: + try: + from flash_attn.flash_attn_interface import ( + flash_attn_varlen_func as flash_attn_unpadded_func, + ) + except ImportError: + flash_attn_unpadded_func = None + +# GatedDeltaNet GPU kernels -- only available with FLA + causal-conv1d +try: + from fla.ops.gated_delta_rule import chunk_gated_delta_rule, fused_recurrent_gated_delta_rule +except ImportError: + chunk_gated_delta_rule = None + fused_recurrent_gated_delta_rule = None +try: + from causal_conv1d import causal_conv1d_fn, causal_conv1d_update +except ImportError: + causal_conv1d_fn = None + causal_conv1d_update = None + +FLA_AVAILABLE = all([chunk_gated_delta_rule, fused_recurrent_gated_delta_rule, + causal_conv1d_fn, causal_conv1d_update]) + + +# =========================================================================== +# Qwen3.5-specific GPU kernels +# =========================================================================== + +class Qwen3_5RMSNormGated(torch.nn.Module): + """RMSNorm + SiLU gate (used in GatedDeltaNet output).""" + + def __init__(self, dim, eps=1e-6): + super().__init__() + self.weight = torch.nn.Parameter(torch.ones(dim)) + self.eps = eps + + def forward(self, x, gate): + # norm(x) * silu(gate) + rms = torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + return (x * rms * self.weight) * F.silu(gate) + + +class Qwen3_5RMSNorm(torch.nn.Module): + """Qwen3.5 RMSNorm with zero-init weight: output = norm(x) * (1 + weight).""" + + def __init__(self, dim, eps=1e-6): + super().__init__() + self.weight = torch.nn.Parameter(torch.zeros(dim)) + self.eps = eps + + def forward(self, x): + rms = torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + return x * rms * (1.0 + self.weight) + + +class Qwen3_5GatedDeltaNetBenchmark(torch.nn.Module): + """ + GatedDeltaNet GPU benchmark harness. + + Operations (all local compute, no TP): + 1. Conv1D (causal depthwise, kernel_dim=4) on fused QKV + 2. Split into Q, K, V + 3. Gated Delta Rule recurrence: S_t = S_{t-1} * decay + k^T * delta + 4. Output gate: z projection, silu activation + 5. RMSNormGated: norm(output) * silu(z) + 6. Output projection: value_dim -> hidden_size + """ + + def __init__(self, args, layer_id=0): + super().__init__() + self.hidden_size = args.hidden_size + self.num_v_heads = getattr(args, "linear_num_value_heads", 32) + self.num_k_heads = getattr(args, "linear_num_key_heads", 16) + self.head_k_dim = getattr(args, "linear_key_head_dim", 128) + self.head_v_dim = getattr(args, "linear_value_head_dim", 128) + self.conv_kernel_size = getattr(args, "linear_conv_kernel_dim", 4) + self.key_dim = self.head_k_dim * self.num_k_heads + self.value_dim = self.head_v_dim * self.num_v_heads + self.layer_id = layer_id + + if args.dtype == "bfloat16": + dtype = torch.bfloat16 + elif args.dtype == "float16": + dtype = torch.float16 + else: + dtype = torch.float32 + device = torch.cuda.current_device() + self.dtype = dtype + + # Input projections + self.in_proj_qkv = torch.rand(self.hidden_size, + self.key_dim * 2 + self.value_dim, + device=device).to(dtype) + self.in_proj_z = torch.rand(self.hidden_size, self.value_dim, + device=device).to(dtype) + self.in_proj_b = torch.rand(self.hidden_size, self.num_v_heads, + device=device).to(dtype) + self.in_proj_a = torch.rand(self.hidden_size, self.num_v_heads, + device=device).to(dtype) + + # Conv1D weight + conv_dim = self.key_dim * 2 + self.value_dim + self.conv_weight = torch.rand(conv_dim, self.conv_kernel_size, + device=device).to(dtype) + + # Output projection + self.out_proj = torch.rand(self.value_dim, self.hidden_size, + device=device).to(dtype) + + # RMSNormGated for output + self.norm = Qwen3_5RMSNormGated(self.head_v_dim, eps=1e-6) + + # Delta rule parameters + self.dt_bias = torch.rand(self.num_v_heads, device=device).to(dtype) + self.A_log = torch.rand(self.num_v_heads, device=device).to(dtype) + + # Input tensor + self.input = torch.rand(args.seq_length, args.micro_batch, + args.hidden_size, device=device).to(dtype) + + def forward(self): + if not FLA_AVAILABLE: + return 0.0 # cannot benchmark without GPU libraries + + x = self.input + seq_len = x.shape[0] + batch = x.shape[1] + + torch.cuda.synchronize() + t_start = time.time() + + # QKV projection + Z projection + B/A projections + mixed_qkv = F.linear(x, self.in_proj_qkv) + z = F.linear(x, self.in_proj_z) + b = F.linear(x, self.in_proj_b) + a = F.linear(x, self.in_proj_a) + + # Causal Conv1D + mixed_qkv = mixed_qkv.transpose(0, 1).transpose(1, 2) # (B, C, L) + mixed_qkv = causal_conv1d_fn( + x=mixed_qkv, weight=self.conv_weight.unsqueeze(1), + bias=None, activation="silu", seq_idx=None, + ) + mixed_qkv = mixed_qkv.transpose(1, 2).transpose(0, 1) # back to (L, B, C) + + # Split Q, K, V + query, key, value = torch.split( + mixed_qkv, [self.key_dim, self.key_dim, self.value_dim], dim=-1, + ) + query = query.view(seq_len, batch, self.num_k_heads, self.head_k_dim) + key = key.view(seq_len, batch, self.num_k_heads, self.head_k_dim) + value = value.view(seq_len, batch, self.num_v_heads, self.head_v_dim) + z = z.view(seq_len, batch, self.num_v_heads, self.head_v_dim) + + # Gated Delta Rule + beta = b.sigmoid() + g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias) + + # Repeat K/Q if num_v_heads > num_k_heads (GQA for DeltaNet) + if self.num_v_heads // self.num_k_heads > 1: + r = self.num_v_heads // self.num_k_heads + query = query.repeat_interleave(r, dim=2) + key = key.repeat_interleave(r, dim=2) + + core_out, _ = chunk_gated_delta_rule( + query, key, value, g=g, beta=beta, + initial_state=None, output_final_state=False, + use_qk_l2norm_in_kernel=True, + ) + + # Output gate + RMSNormGated + output projection + core_out = core_out.reshape(-1, self.head_v_dim) + z = z.reshape(-1, self.head_v_dim) + gated = self.norm(core_out, z) + output = F.linear(gated, self.out_proj) + + torch.cuda.synchronize() + return (time.time() - t_start) * 1e6 # microseconds + + +class Qwen3_5GatedAttentionBenchmark(torch.nn.Module): + """ + Qwen3.5 gated full-attention benchmark (25% of layers). + + Differences from Qwen3 attention: + - head_dim = 256 (2x Qwen3) + - q_proj outputs 2 * num_heads * head_dim (half query, half gate) + - sigmoid gate on attention output + - QK-Norm per-head (RMSNorm on 256) + - partial RoPE: only rotary_dim = head_dim * partial_rotary_factor = 64 dims + """ + + def __init__(self, args): + super().__init__() + self.tp = args.tensor_model_parallel_size + hidden_size = args.hidden_size + n_heads = args.num_attention_heads + n_kv = args.num_key_value_heads + head_dim = getattr(args, "head_dim", 256) + micro_batch = args.micro_batch + seq_len = args.seq_length + + if args.dtype == "bfloat16": + dtype = torch.bfloat16 + elif args.dtype == "float16": + dtype = torch.float16 + else: + dtype = torch.float32 + device = torch.cuda.current_device() + self.dtype = dtype + self.head_dim = head_dim + + # Doubled q_proj: half query, half gate + q_dim = n_heads * head_dim * 2 + self.q_weight = torch.rand(divide(q_dim, self.tp), hidden_size, + device=device).to(dtype) + self.k_weight = torch.rand(divide(n_kv * head_dim, self.tp), hidden_size, + device=device).to(dtype) + self.v_weight = torch.rand(divide(n_kv * head_dim, self.tp), hidden_size, + device=device).to(dtype) + self.o_weight = torch.rand(hidden_size, divide(n_heads * head_dim, self.tp), + device=device).to(dtype) + + # QK-Norm + self.q_norm = Qwen3_5RMSNorm(head_dim, eps=1e-6) + self.k_norm = Qwen3_5RMSNorm(head_dim, eps=1e-6) + + # Input + self.input = torch.rand(seq_len, micro_batch, hidden_size, + device=device).to(dtype) + + def forward(self): + x = self.input + + torch.cuda.synchronize() + t_start = time.time() + + # Q projection (doubled) + K + V + q_out = F.linear(x, self.q_weight) # (L, B, 2 * heads * head_dim / tp) + k_out = F.linear(x, self.k_weight) + v_out = F.linear(x, self.v_weight) + + # Split query and gate + q_dim = q_out.shape[-1] // 2 + q_raw = q_out[:, :, :q_dim] + gate = q_out[:, :, q_dim:] + + # QK-Norm (per-head) + # Reshape: (L, B, heads/tp, head_dim) + q_normed = self.q_norm(q_raw) + k_normed = self.k_norm(k_out) + + # Output projection + gate + o_out = F.linear(q_normed, self.o_weight) + o_out = o_out * torch.sigmoid(gate) + + torch.cuda.synchronize() + return (time.time() - t_start) * 1e6 + + +class Qwen3_5MlpBenchmark(torch.nn.Module): + """Qwen3.5 SwiGLU MLP (same as Qwen3, different intermediate_size).""" + + def __init__(self, args): + super().__init__() + self.tp = args.tensor_model_parallel_size + hidden_size = args.hidden_size + intermediate_size = getattr(args, "intermediate_size", + getattr(args, "ffn_hidden_size", 4 * hidden_size)) + swiglu = getattr(args, "swiglu", True) + if args.dtype == "bfloat16": + dtype = torch.bfloat16 + elif args.dtype == "float16": + dtype = torch.float16 + else: + dtype = torch.float32 + device = torch.cuda.current_device() + self.dtype = dtype + + gate_up_dim = 2 * intermediate_size if swiglu else intermediate_size + self.gate_up_weight = torch.rand(divide(gate_up_dim, self.tp), hidden_size, + device=device).to(dtype) + self.down_weight = torch.rand(hidden_size, divide(intermediate_size, self.tp), + device=device).to(dtype) + self.input = torch.rand(args.seq_length, args.micro_batch, + hidden_size, device=device).to(dtype) + + def forward(self): + x = self.input + torch.cuda.synchronize() + t_start = time.time() + gate_up = F.linear(x, self.gate_up_weight) + gate_up = F.silu(gate_up) + down = F.linear(gate_up, self.down_weight) + torch.cuda.synchronize() + return (time.time() - t_start) * 1e6 + + +# =========================================================================== +# Full Qwen3.5 benchmark model +# =========================================================================== + +class Qwen3_5Model(torch.nn.Module): + """Complete Qwen3.5 training benchmark with hybrid GatedDeltaNet + Full Attention.""" + + def __init__(self, args): + super().__init__() + self.time_list = {} + self.args = args + + from workload_generator.mocked_model.training.AiobMegatron import ( + MegatronEmbedding, MegatronLayernorm, logit, Grad_param, + ) + + self.Embedding = MegatronEmbedding(self.args) + self.Layernorm = MegatronLayernorm(self.args) + + num_layers = getattr(args, "num_hidden_layers", args.num_layers) + full_interval = getattr(args, "full_attention_interval", 4) + + self.gdn_layers = [] + self.attn_layers = [] + self.mlp_layers = [] + for i in range(num_layers): + if (i + 1) % full_interval == 0: + self.attn_layers.append(Qwen3_5GatedAttentionBenchmark(self.args)) + else: + self.gdn_layers.append(Qwen3_5GatedDeltaNetBenchmark(self.args, layer_id=i)) + self.mlp_layers.append(Qwen3_5MlpBenchmark(self.args)) + + self.logit = logit(self.args) + self.grad_param = Grad_param(self.args) + + def forward(self, input_tensor): + for _ in range(self.args.epoch_num): + Emb_output, Emb_time = self.Embedding(input_tensor) + self.time_list.setdefault("Emb", []).append({"time_gpu": Emb_time}) + + gdn_idx = 0 + attn_idx = 0 + for i in range(len(self.mlp_layers)): + lay_out, lay_time = self.Layernorm(Emb_output) + self.time_list.setdefault("layernorm", []).append({"time_gpu": lay_time}) + + # Token mixer + if (i + 1) % getattr(self.args, "full_attention_interval", 4) == 0: + attn_time = self.attn_layers[attn_idx]() + attn_idx += 1 + self.time_list.setdefault("atten_full", []).append({"time_gpu": attn_time}) + else: + if FLA_AVAILABLE: + gdn_time = self.gdn_layers[gdn_idx]() + gdn_idx += 1 + self.time_list.setdefault("atten_gdn", []).append({"time_gpu": gdn_time}) + else: + self.time_list.setdefault("atten_gdn", []).append({"time_gpu": 1}) + + # MLP + lay_out2, lay_time2 = self.Layernorm(lay_out) + self.time_list.setdefault("layernorm2", []).append({"time_gpu": lay_time2}) + mlp_time = self.mlp_layers[i]() + self.time_list.setdefault("mlp", []).append({"time_gpu": mlp_time}) + + # Final + lay_out_post, post_time = self.Layernorm(Emb_output) + self.time_list.setdefault("layernorm_post", []).append({"time_gpu": post_time}) + logit_time = self.logit(lay_out_post) + self.time_list.setdefault("logit_time", []).append({"time_gpu": logit_time}) + grad_time = self.grad_param(lay_out_post) + self.time_list.setdefault("param_time", []).append({"time_gpu": grad_time}) + + from utils.utils import write_time + return write_time(self.time_list, self.args) diff --git a/aicb/workload_generator/mocked_model/training/MockedDeepSeek.py b/aicb/workload_generator/mocked_model/training/MockedDeepSeek.py new file mode 100644 index 000000000..5b3764828 --- /dev/null +++ b/aicb/workload_generator/mocked_model/training/MockedDeepSeek.py @@ -0,0 +1,537 @@ +""" +Provides DeepSeek implementation for MockedModel + +Based on https://github.com/deepseek-ai/DeepSeek-V3/tree/f6e34dd26772dd4a216be94a8899276c5dca9e43 + +@misc{deepseekai2024deepseekv3technicalreport, + title={DeepSeek-V3 Technical Report}, + author={DeepSeek-AI}, + year={2024}, + eprint={2412.19437}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2412.19437}, +} + + +File: MockedDeepSeek.py +License: Apache 2.0 +""" + +from workload_generator.mocked_model.MockedModel import MockedModel +from workload_generator.mocked_model.training.MockedMegatron import * +from utils.utils import CommGroup, CommType + +# multiplier to convert BF16 to (FP8 + FP32 scale) +# from https://github.com/deepseek-ai/DeepEP/blob/ef70b83e3b35a84aadc5385b02c95c5d1bcf299c/tests/test_internode.py#L194 +FP8_FACTOR = (1 + 4 / 128) / 2 + +class DeepSeekLinear(MockedModel): + """ + non-sharded Linear for DeepSeek + + Attributes: + in_feature (int): input dimention + out_feature (int): output dimention + computation_enable (bool): if True, add compute LogItem + name (str): layer name + bias (bool): if ture, add bias term. Defaults to False + // maybe don't need these + seq_len (int): + batch_size (int): + """ + + def __init__( + self, + in_feature: int, + out_feature: int, + computation_enable: bool, + seq_len: int, + batch_size: int, + layer_id: int, + name: str = "", + bias: bool = False, + ): + super().__init__() + self.in_feature = in_feature + self.out_feature = out_feature + self.name = name + self.seq_len = seq_len + self.batch_size = batch_size + self.layer_id = layer_id + self.w = MockedParam( + (self.in_feature, self.out_feature), name=self.name + "_linear" + ) + if bias: + self.bias = MockedParam((out_feature, 1), name=self.name + "_bias") + self.computation_enable = computation_enable + + def forward(self): + workloads = Workload() + if self.computation_enable: + workloads.append( + LogItem( + comm_type=CommType.computation, + msg_size=( + (self.seq_len, self.batch_size, self.in_feature), + (self.in_feature, self.out_feature), + ), + stage=f"forward.Linear.{self.name}", + ) + ) + return workloads + + def backward(self): + workloads = Workload() + if self.computation_enable: + workloads.append( + LogItem( + comm_type=CommType.computation, + msg_size=( + (self.out_feature, self.seq_len * self.batch_size), + (self.seq_len * self.batch_size, self.in_feature), + ), + stage=f"backward.Linear.{self.name}", + ) + ) + return workloads + + +class DeepSeekMLA(MockedModel): + """Multi Latent Attentnion layer for DeepSeek""" + + def __init__( + self, + num_attention_heads, + hidden_size, + tp, + seq_len, + batch_size, + layer_id, + sequence_parallel_enabled, + computation_enable, + add_bias_linear, + qk_rope_dim, + qk_nope_dim, + v_head_dim, + q_lora_rank, + kv_lora_rank, + ): + self.name = "attention_layer_mla" + self.layer_id = layer_id + self.qk_dim = qk_nope_dim + qk_rope_dim + self.v_head_dim = v_head_dim + + self.kv_channels = hidden_size // num_attention_heads + self.kv_projection_size = self.kv_channels * num_attention_heads + self.query_projection_size = self.kv_channels * num_attention_heads + + self.q_lora_rank = q_lora_rank + self.kv_lora_rank = kv_lora_rank + + # Q down projection + self.wq_a = DeepSeekLinear( + in_feature=hidden_size, + out_feature=self.q_lora_rank, + computation_enable=computation_enable, + name="attention_linear_q_lora", + bias=add_bias_linear, + layer_id=layer_id, + batch_size=batch_size, + seq_len=seq_len, + ) + self.q_norm = FusedLayernorm(self.q_lora_rank) + self.wq_b = MegatronColumnLinear( + self.q_lora_rank, + (num_attention_heads) * self.qk_dim, + tp, + seq_len, + batch_size, + layer_id, + "attention_q", + sequence_parallel_enabled, + computation_enable, + name="attention_q_column", + add_bias_linear=add_bias_linear, + ) + + # KV down projection + self.wkv_a = DeepSeekLinear( + in_feature=hidden_size, + out_feature=self.kv_lora_rank + qk_rope_dim, + computation_enable=computation_enable, + name="attention_linear_kv_lora", + bias=add_bias_linear, + layer_id=layer_id, + batch_size=batch_size, + seq_len=seq_len, + ) + self.kv_norm = FusedLayernorm(self.kv_lora_rank) + self.wkv_b = MegatronColumnLinear( + self.kv_lora_rank, + num_attention_heads * (qk_nope_dim + self.v_head_dim), + tp, + seq_len, + batch_size, + layer_id, + "attention_kv", + sequence_parallel_enabled, + computation_enable, + name="attention_kv_column", + add_bias_linear=add_bias_linear, + ) + self.wo = MegatronRowLinear( + num_attention_heads * self.v_head_dim, + hidden_size, + tp, + seq_len, + batch_size, + layer_id, + "attention_o", + sequence_parallel_enabled, + computation_enable, + name="attention_o_row", + add_bias_linear=add_bias_linear, + ) + + def forward(self): + workloads = Workload() + # extend for Linear parts + + # for down projected Q + # from deepseek.py: https://github.com/deepseek-ai/DeepSeek-V3/blob/f6e34dd26772dd4a216be94a8899276c5dca9e43/inference/model.py#L461C1-L461C53 + # q = self.wq_b(self.q_norm(self.wq_a(x))) + workloads.extend(self.wq_a.forward()) + # since we don't have a forward() impl for norm, thus skip norm + # workloads.extend(self.q_norm.forward()) + + workloads.extend(self.wq_b.forward()) + # add RoPE (ommited here) + + # for down projected KV + workloads.extend(self.wkv_a.forward()) + # similarly for kv_norm, we skip norm because no forward() impl for norm + # workloads.extend(self.kv_norm.forward()) + workloads.extend(self.wkv_b.forward()) + + # for O + workloads.extend(self.wo.forward()) + assert all([isinstance(workload, LogItem) for workload in workloads.workload]) + return workloads + + def backward(self): + workloads = Workload() + # similar to fwd but .backward() + workloads.extend(self.wq_a.backward()) + workloads.extend(self.wq_b.backward()) + workloads.extend(self.wkv_a.backward()) + workloads.extend(self.wkv_b.backward()) + workloads.extend(self.wo.backward()) + assert all([isinstance(workload, LogItem) for workload in workloads.workload]) + return workloads + + +class DeepSeekMoE(MockedModel): + def __init__( + self, + batch_size, + hidden_size, + tp, + expert_model_parallel_size, + ffn_hidden_size, + seq_len, + topk, + num_experts, + id, + n_shared_expert, + sequence_parallel_enabled, + computation_enable, + add_bias_linear, + ): + self.name = "mlp_moelayer" + self.layer_id = id + self.name = "mlp_moelayer" + self.layer_id = id + self.expert_model_parallel_size = expert_model_parallel_size + num_local_experts = num_experts // expert_model_parallel_size + fc1_output_size = ffn_hidden_size * num_local_experts + fc1_output_size_per_parttition = divide(fc1_output_size, tp) + fc2_input_size = ffn_hidden_size * num_local_experts + fc2_input_size_per_parttition = divide(fc2_input_size, tp) + self.weight1 = MockedParam((hidden_size, fc1_output_size_per_parttition)) + self.weight2 = MockedParam((fc2_input_size_per_parttition, hidden_size)) + self.weight3 = MockedParam((hidden_size, fc2_input_size_per_parttition)) + self.tp_size = tp + self.topk = topk + self.seq_len = seq_len + self.num_experts = num_experts + self.batch_size = batch_size + self.hidden_size = hidden_size + self.shared_experts = None + if n_shared_expert > 0: + self.shared_experts = MegatronMlp( + hidden_size, + ffn_hidden_size, + tp, + seq_len, + batch_size, + id, + sequence_parallel_enabled, + computation_enable, + add_bias_linear, + ) + + def permutation(self, stage): + workloads = Workload() + if self.expert_model_parallel_size > 1: + # only for FWD + # FP8 dispatch, include input/128 matrix for scale + # based on DeepEP https://github.com/parthpower/DeepEP/commit/50aee15f592bc22142eb04b7d718296b19613ae9 + if stage == "forward": + scaled = FP8_FACTOR + else: + scaled = 1 + workloads.append( + LogItem( + comm_type=CommType.all_to_all, + comm_group=CommGroup.ep_group, + msg_size=( + self.seq_len * self.hidden_size * self.batch_size * self.topk // self.tp_size // self.expert_model_parallel_size + ) + * 2 + * scaled, + stage=f"{stage}.MoE.dispatch", + ) + ) + if self.tp_size > 1: + # Dividing by ep_size: after dispatch each EP rank holds ~1/ep of tokens + workloads.append( + LogItem( + comm_type=CommType.all_gather, + comm_group=CommGroup.tp_group, + msg_size=( + self.hidden_size * self.topk * self.batch_size * self.seq_len // self.expert_model_parallel_size + ) + * 2, + stage=f"{stage}.MoE.permutation", + ) + ) + + return workloads + + def unpermutation(self, stage): + workloads = Workload() + if self.tp_size > 1: + # Dividing by ep_size: after dispatch each EP rank holds ~1/ep of tokens + workloads.append( + LogItem( + comm_type=CommType.reduce_scatter, + comm_group=CommGroup.tp_group, + msg_size=self.hidden_size + * self.batch_size + * self.topk + * self.seq_len + * 2 // self.expert_model_parallel_size, + stage=f"{stage}.MoE.unpermutation", + ) + ) + + if self.expert_model_parallel_size > 1: + # bf16 all-to-all combine + workloads.append( + LogItem( + comm_type=CommType.all_to_all, + comm_group=CommGroup.ep_group, + msg_size=( + self.seq_len * self.hidden_size * self.batch_size * self.topk // self.tp_size // self.expert_model_parallel_size + ) + * 2, + stage=f"{stage}.MoE.combine", + ) + ) + + return workloads + + def moe_mlp_forward(self): + workloads = Workload() + workloads.extend(self.permutation(stage="forward")) + workloads.extend(self.unpermutation(stage="forward")) + assert all([isinstance(workload, LogItem) for workload in workloads.workload]) + return workloads + + def moe_mlp_backward(self): + workloads = Workload() + workloads.extend(self.permutation(stage="backward")) + workloads.extend(self.unpermutation(stage="backward")) + assert all([isinstance(workload, LogItem) for workload in workloads.workload]) + return workloads + + def forward(self): + wl = Workload() + wl.extend(self.moe_mlp_forward()) + if self.shared_experts != None: + wl.extend(self.shared_experts.forward()) + return wl + + def backward(self): + wl = Workload() + if self.shared_experts != None: + wl.extend(self.shared_experts.backward()) + wl.extend(self.moe_mlp_backward()) + return wl + + +class DeepSeekTransformer(MockedModel): + def __init__( + self, + num_attention_heads, + hidden_size, + tp, + seq_len, + batch_size, + layer_id, + sequence_parallel_enabled, + computation_enable, + add_bias_linear, + qk_rope_dim, + qk_nope_dim, + v_head_dim, + expert_model_parallel_size, + ffn_hidden_size, + moe_router_topk, + num_experts, + n_shared_expert, + q_lora_rank, + kv_lora_rank, + use_dense, + ): + self.attention = DeepSeekMLA( + num_attention_heads, + hidden_size, + tp, + seq_len, + batch_size, + layer_id, + sequence_parallel_enabled, + computation_enable, + add_bias_linear, + qk_rope_dim, + qk_nope_dim, + v_head_dim, + q_lora_rank, + kv_lora_rank, + ) + self.pre_mlp_layernorm = FusedLayernorm(hidden_size) + self.post_attention_layernorm_bias = MockedParam((hidden_size, 1)) + if use_dense: + self.mlp = MegatronMlp( + hidden_size, + ffn_hidden_size, + tp, + seq_len, + batch_size, + layer_id, + sequence_parallel_enabled, + computation_enable, + add_bias_linear, + ) + else: + self.mlp = DeepSeekMoE( + batch_size, + hidden_size, + tp, + expert_model_parallel_size, + ffn_hidden_size, + seq_len, + moe_router_topk, + num_experts, + layer_id, + n_shared_expert, + sequence_parallel_enabled, + computation_enable, + add_bias_linear, + ) + + def forward(self): + worklods = Workload() + worklods.extend(self.attention.forward()) + worklods.extend(self.mlp.forward()) + return worklods + + def backward(self): + workloads = Workload() + workloads.extend(self.attention.backward()) + workloads.extend(self.mlp.backward()) + return workloads + + +class DeepSeekV3Model(MockedModel): + def __init__(self, config): + self.embedding = MegatronEmbedding( + config.padded_vocab_size, + config.hidden_size, + config.tensor_model_parallel_size, + config.seq_length, + config.micro_batch, + ) + self.layers = [ + DeepSeekTransformer( + num_attention_heads=config.num_attention_heads, + hidden_size=config.hidden_size, + add_bias_linear=config.add_bias_linear, + batch_size=config.micro_batch, + computation_enable=config.computation_enable, + expert_model_parallel_size=config.expert_model_parallel_size, + ffn_hidden_size=config.ffn_hidden_size, + layer_id=id, + moe_router_topk=config.moe_router_topk, + n_shared_expert=config.n_shared_expert, + num_experts=config.num_experts, + qk_nope_dim=config.qk_nope_dim, # new + qk_rope_dim=config.qk_rope_dim, # new + seq_len=config.seq_length, + sequence_parallel_enabled=config.enable_sequence_parallel, + tp=config.tensor_model_parallel_size, + q_lora_rank=config.q_lora_rank, # new + kv_lora_rank=config.kv_lora_rank, # new + v_head_dim=config.v_head_dim, # new + use_dense=(id < config.n_dense_layers), + ) + for id in range(config.num_layers) + ] + + self.final_norm = MegatronColumnLinear( + config.hidden_size, + config.padded_vocab_size, + config.tensor_model_parallel_size, + config.seq_length, + config.micro_batch, + 1, + "final", + sequence_parallel_enabled=config.enable_sequence_parallel, + computation_enable=config.computation_enable, + add_bias_linear=config.add_bias_linear, + ) + + def forward(self): + workloads = Workload() + workloads.extend(self.embedding.forward()) + for layer in self.layers: + fwd = layer.forward() + if not isinstance(fwd, Workload): + continue + workloads.extend(fwd) + + assert all([isinstance(workload, LogItem) for workload in workloads.workload]) + return workloads + + def backward(self): + workloads = Workload() + for layer in self.layers[::-1]: + bwd = layer.backward() + if not isinstance(bwd, Workload): + continue + workloads.extend(bwd) + workloads.extend(self.embedding.backward()) + assert all([isinstance(workload, LogItem) for workload in workloads.workload]) + return workloads diff --git a/aicb/workload_generator/mocked_model/training/MockedDeepspeed.py b/aicb/workload_generator/mocked_model/training/MockedDeepspeed.py new file mode 100644 index 000000000..289f8acd1 --- /dev/null +++ b/aicb/workload_generator/mocked_model/training/MockedDeepspeed.py @@ -0,0 +1,57 @@ +""" +Copyright (c) 2021, Alibaba Group; +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from typing import List +from workload_generator.mocked_model.MockedModel import MockedModel, Linear + + +class DeepspeedMLP(MockedModel): + def __init__(self, hidden_size: int, ffn_hidden_size: int): + self.gate_proj = Linear(hidden_size, ffn_hidden_size) + self.down_proj = Linear(ffn_hidden_size, hidden_size) + self.up_proj = Linear(hidden_size, ffn_hidden_size) + + +class DeepspeedAttention(MockedModel): + def __init__(self, config): + self.hidden_size = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.hidden_size // self.num_heads + self.max_position_embeddings = config.max_position_embeddings + self.q_proj = Linear(self.hidden_size, self.num_heads * self.head_dim) + self.k_proj = Linear(self.hidden_size, self.num_heads * self.head_dim) + self.v_proj = Linear(self.hidden_size, self.num_heads * self.head_dim) + self.o_proj = Linear(self.num_heads * self.head_dim, self.hidden_size) + # self.rotary_emb = Linear(self.head_dim, self.max_position_embeddings) + + +class DeepspeedDecoderLayer(MockedModel): + def __init__(self, config): + self.input_layernorm = Linear(config.hidden_size, 1) + self.self_attn = DeepspeedAttention(config=config) + self.post_attention_layernorm = Linear(config.hidden_size, 1) + self.mlp = DeepspeedMLP(config.hidden_size, config.ffn_hidden_size) + + +class DeepspeedModel(MockedModel): + def __init__(self, config): + self.embed_tokens = Linear(config.vocab_size, config.hidden_size) + self.layers = [DeepspeedDecoderLayer(config) for _ in range(config.num_layers)] + self.norm = Linear(config.hidden_size, 1) + + +class DeepspeedForCausalLM(MockedModel): + def __init__(self, config): + super().__init__() + self.model = DeepspeedModel(config) + self.lm_head = Linear(config.hidden_size, config.vocab_size) diff --git a/aicb/workload_generator/mocked_model/training/MockedMegatron.py b/aicb/workload_generator/mocked_model/training/MockedMegatron.py new file mode 100755 index 000000000..8fb9c99fb --- /dev/null +++ b/aicb/workload_generator/mocked_model/training/MockedMegatron.py @@ -0,0 +1,691 @@ +""" +Copyright (c) 2021, Alibaba Group; +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from utils.utils import divide, CommType, CommGroup +from workload_generator.mocked_model.MockedModel import MockedModel, Linear, MockedParam +from log_analyzer.log import Workload, LogItem + + +# mocked version of Megatron RowParallelLinear +class MegatronRowLinear(MockedModel): + def __init__( + self, + input_size, + output_size, + tp, + seq_len, + batch_size, + layer_id, + prefix_name, + sequence_parallel_enabled=True, + computation_enable=False, + name=None, + add_bias_linear=False, + ): + self.layer_id = layer_id + self.name = prefix_name + "_row" + self.input_size, self.output_size = input_size, output_size + self.input_size_per_partition = divide(input_size, tp) + self.weight = MockedParam( + (output_size, self.input_size_per_partition), name=name + ) + if add_bias_linear: + self.bias = MockedParam((output_size, 1), name=self.name + "_bias") + self.sequence_parallel_enabled = sequence_parallel_enabled + self.computation_enable = computation_enable + self.tensor_model_parallel_size, self.seq_len, self.batch_size = tp, seq_len, batch_size + self.comm_size = 2 * seq_len * batch_size * output_size + + def activation_memory(self): + # ctx.save_for_backward(input, weight) + return self.seq_len * self.input_size + + def forward(self): + workloads = Workload() + # output_ = torch.matmul(total_input, weight.t()): (s, b, h) + if self.computation_enable: + workloads.append( + LogItem( + comm_type=CommType.computation, + msg_size=( + (self.seq_len, self.batch_size, self.input_size_per_partition), + (self.input_size_per_partition, self.output_size), + ), + stage="forward.MegatronRowLinear." + self.name, + ) + ) + if self.tensor_model_parallel_size > 1: + if self.sequence_parallel_enabled: + # output_ = reduce_scatter_to_sequence_parallel_region(output_parallel): (s/tp, b, h) + workloads.append( + LogItem( + comm_type=CommType.reduce_scatter, + comm_group=CommGroup.tp_group, + comm_group_size=self.tensor_model_parallel_size, + msg_size=self.comm_size, + stage="forward.MegatronRowLinear", + ) + ) + else: + # output_ = reduce_from_tensor_model_parallel_region(output_parallel) + workloads.append( + LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.tp_group, + comm_group_size=self.tensor_model_parallel_size, + msg_size=self.comm_size, + stage="forward.MegatronRowLinear", + ) + ) + return workloads + + def backward(self): + workloads = Workload() + if self.tensor_model_parallel_size > 1: + if self.sequence_parallel_enabled: + # output_ = reduce_scatter_to_sequence_parallel_region(output_parallel): (s/tp, b, h) + workloads.append( + LogItem( + comm_type=CommType.all_gather, + comm_group=CommGroup.tp_group, + comm_group_size=self.tensor_model_parallel_size, + msg_size=self.comm_size, + stage="backward.MegatronRowLinear", + ) + ) + # grad_input = grad_output.matmul(weight): (s, b, h)*(h, h'/N) + # grad_weight = grad_output.t().matmul(total_input): (h, s*b)*(s*b, h'/N) + if self.computation_enable: + workloads.append( + LogItem( + comm_type=CommType.computation, + msg_size=( + (self.seq_len, self.batch_size, self.output_size), + self.weight.shape, + ), + stage="backward.MegatronRowLinear" + self.name, + ) + ) + workloads.append( + LogItem( + comm_type=CommType.computation, + msg_size=( + (self.output_size, self.seq_len * self.batch_size), + (self.seq_len * self.batch_size, self.input_size_per_partition), + ), + stage="backward.MegatronRowLinear" + self.name, + ) + ) + return workloads + + +class MegatronColumnLinear(MockedModel): + def __init__( + self, + input_size, + output_size, + tp, + seq_len, + batch_size, + layer_id, + prefix_name, + sequence_parallel_enabled=True, + computation_enable=False, + name=None, + add_bias_linear=False, + ): + self.layer_id = layer_id + self.name = prefix_name + "_column" + self.input_size, self.output_size = input_size, output_size + self.output_size_per_partition = divide(output_size, tp) + self.weight = MockedParam( + (input_size , self.output_size_per_partition), name=name + ) + if add_bias_linear: + self.bias = MockedParam( + (self.output_size_per_partition, 1), name=self.name + "_bias" + ) + self.sequence_parallel_enabled = sequence_parallel_enabled + self.computation_enable = computation_enable + self.tensor_model_parallel_size, self.seq_len, self.batch_size = tp, seq_len, batch_size + self.comm_size = 2 * seq_len * batch_size * input_size + if self.tensor_model_parallel_size > 1 and self.sequence_parallel_enabled: + self.seq_len *= self.tensor_model_parallel_size + + def activation_memory(self): + # ctx.save_for_backward(input, weight) + return self.seq_len * self.input_size + + def forward(self): + workloads = Workload() + if self.tensor_model_parallel_size > 1: + if self.sequence_parallel_enabled: + workloads.append( + LogItem( + comm_type=CommType.all_gather, + comm_group=CommGroup.tp_group, + comm_group_size=self.tensor_model_parallel_size, + msg_size=self.comm_size, + stage="forward.MegatronColumnLinear", + ) + ) + # output = torch.matmul(total_input, weight.t()) + if self.computation_enable: + workloads.append( + LogItem( + comm_type=CommType.computation, + msg_size=( + (self.seq_len, self.batch_size, self.input_size), + (self.input_size, self.output_size_per_partition), + ), + stage="forward.MegatronColumnLinear." + self.name, + ) + ) + return workloads + + def backward(self): + workloads = Workload() + if self.tensor_model_parallel_size > 1: + if self.sequence_parallel_enabled: + workloads.append( + LogItem( + comm_type=CommType.all_gather, + comm_group=CommGroup.tp_group, + comm_group_size=self.tensor_model_parallel_size, + msg_size=self.comm_size, + stage="backward.MegatronColumnLinear", + ) + ) + # grad_input = grad_output.matmul(weight): (s, b, h'/N)*(h'/N, h) + # grad_weight = grad_output.t().matmul(total_input): (h, s*b)*(s*b, h'/N) + if self.computation_enable: + workloads.append( + LogItem( + comm_type=CommType.computation, + msg_size=( + (self.seq_len, self.batch_size, self.output_size_per_partition), + (self.output_size_per_partition, self.input_size), + ), + stage="backward.MegatronColumnLinear." + self.name, + ) + ) + if self.tensor_model_parallel_size > 1: + if self.sequence_parallel_enabled: + workloads.append( + LogItem( + comm_type=CommType.reduce_scatter, + comm_group=CommGroup.tp_group, + comm_group_size=self.tensor_model_parallel_size, + msg_size=self.comm_size, + stage="backward.MegatronColumnLinear", + ) + ) + if self.computation_enable: + workloads.append( + LogItem( + comm_type=CommType.computation, + msg_size=( + ( + self.output_size_per_partition, + self.seq_len * self.batch_size, + ), + (self.seq_len * self.batch_size, self.input_size), + ), + stage="backward.MegatronColumnLinear." + self.name, + ) + ) + if self.tensor_model_parallel_size > 1: + if not self.sequence_parallel_enabled: + workloads.append( + LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.tp_group, + comm_group_size=self.tensor_model_parallel_size, + msg_size=self.comm_size, + stage="backward.MegatronColumnLinear", + ) + ) + return workloads + + +class FusedLayernorm(MockedModel): + def __init__(self, hidden_size): + self.layer_id = 0 + self.name = "fused" + self.weight = MockedParam((hidden_size, 1)) + self.bias = MockedParam((hidden_size, 1)) + + +class MegatronAttention(MockedModel): + def __init__( + self, + num_attention_heads, + hidden_size, + tp, + seq_len, + batch_size, + layer_id, + sequence_parallel_enabled, + computation_enable, + add_bias_linear, + ): + self.name = "attention_layer" + self.layer_id = layer_id + self.kv_channels = hidden_size // num_attention_heads + self.kv_projection_size = self.kv_channels * num_attention_heads + self.query_projection_size = self.kv_channels * num_attention_heads + self.qkv = MegatronColumnLinear( + hidden_size, + self.query_projection_size + 2 * self.kv_projection_size, + tp, + seq_len, + batch_size, + layer_id, + "attention", + sequence_parallel_enabled, + computation_enable, + name="attention_column", + add_bias_linear=add_bias_linear, + ) + self.attention_dense = MegatronRowLinear( + self.query_projection_size, + hidden_size, + tp, + seq_len, + batch_size, + layer_id, + "attention", + sequence_parallel_enabled, + computation_enable, + name="attention_row", + add_bias_linear=add_bias_linear, + ) + + def forward(self): + workloads = Workload() + workloads.extend(self.qkv.forward()) + workloads.extend(self.attention_dense.forward()) + assert all([isinstance(workload, LogItem) for workload in workloads.workload]) + return workloads + + def backward(self): + workloads = Workload() + workloads.extend(self.qkv.backward()) + workloads.extend(self.attention_dense.backward()) + assert all([isinstance(workload, LogItem) for workload in workloads.workload]) + return workloads + + +class MegatronMlp(MockedModel): + """Standard or SwiGLU MLP. + + When swiglu=False (standard FFN): intermediate_size = 4 * hidden_size. + ColumnLinear: hidden -> intermediate_size. RowLinear same. + + When swiglu=True: intermediate_size is the SwiGLU intermediate dim. + ColumnLinear: hidden -> 2 * intermediate_size (gate_proj + up_proj fused). + RowLinear: intermediate_size -> hidden (down_proj, NOT 2×). + """ + + def __init__( + self, + hidden_size, + intermediate_size, + tp, + seq_len, + batch_size, + layer_id, + sequence_parallel_enabled, + computation_enable, + add_bias_linear, + swiglu=False, + ): + self.name = "mlp_layer" + self.layer_id = layer_id + + gate_up_dim = 2 * intermediate_size if swiglu else intermediate_size + down_dim = intermediate_size + + self.dense_h_to_4h = MegatronColumnLinear( + hidden_size, + gate_up_dim, + tp, seq_len, batch_size, + layer_id, "mlp", + sequence_parallel_enabled, computation_enable, + name="mlp_column", + add_bias_linear=add_bias_linear, + ) + self.dense_4h_to_h = MegatronRowLinear( + down_dim, + hidden_size, + tp, seq_len, batch_size, + layer_id, "mlp", + sequence_parallel_enabled, computation_enable, + name="mlp_row", + add_bias_linear=add_bias_linear, + ) + + def forward(self): + workloads = Workload() + workloads.extend(self.dense_h_to_4h.forward()) + workloads.extend(self.dense_4h_to_h.forward()) + assert all([isinstance(workload, LogItem) for workload in workloads.workload]) + return workloads + + def backward(self): + workloads = Workload() + workloads.extend(self.dense_h_to_4h.backward()) + workloads.extend(self.dense_4h_to_h.backward()) + assert all([isinstance(workload, LogItem) for workload in workloads.workload]) + return workloads + + +class MOEMLP(MockedModel): + def __init__( + self, + batch_size, + hidden_size, + tp, + expert_model_parallel_size, + ffn_hidden_size, + seq_len, + topk, + num_experts, + id, + ): + self.name = "mlp_moelayer" + self.layer_id = id + num_local_experts = num_experts // expert_model_parallel_size + fc1_output_size = ffn_hidden_size * num_local_experts + fc1_output_size_per_parttition = divide(fc1_output_size, tp) + fc2_input_size = ffn_hidden_size * num_local_experts + fc2_input_size_per_parttition = divide(fc2_input_size, tp) + self.weight1 = MockedParam((hidden_size, fc1_output_size_per_parttition)) + self.weight2 = MockedParam((fc2_input_size_per_parttition, hidden_size)) + self.tp_size = tp + self.ep_size = expert_model_parallel_size + self.topk = topk + self.seq_len = seq_len + self.num_experts = num_experts + self.batch_size = batch_size + self.hidden_size = hidden_size + + def permutation(self, stage): + workloads = Workload() + if self.ep_size > 1: + workloads.append( + LogItem( + comm_type=CommType.all_to_all, + comm_group=CommGroup.ep_group, + msg_size=( + self.seq_len * self.hidden_size * self.batch_size * self.topk // self.tp_size // self.ep_size + ) + * 2, + stage=f"{stage}.MoE.dispatch", + ) + ) + if self.tp_size > 1: + # Dividing by ep_size: after dispatch each EP rank holds ~1/ep of tokens + workloads.append( + LogItem( + comm_type=CommType.all_gather, + comm_group=CommGroup.tp_group, + msg_size=2 + * self.hidden_size + * self.topk * self.batch_size + * self.seq_len // self.ep_size, + stage=f"{stage}.MoE.permutation", + ) + ) + + return workloads + + def unpermutation(self, stage): + workloads = Workload() + if self.tp_size > 1: + # Dividing by ep_size: after dispatch each EP rank holds ~1/ep of tokens + workloads.append( + LogItem( + comm_type=CommType.reduce_scatter, + comm_group=CommGroup.tp_group, + msg_size=2 + * self.hidden_size * self.batch_size + * self.topk + * self.seq_len // self.ep_size, + stage=f"{stage}.MoE.unpermutation", + ) + ) + + if self.ep_size > 1: + workloads.append( + LogItem( + comm_type=CommType.all_to_all, + comm_group=CommGroup.ep_group, + msg_size=( + self.seq_len * self.hidden_size * self.batch_size * self.topk // self.tp_size // self.ep_size + ) + * 2, + stage=f"{stage}.MoE.combine", + ) + ) + + return workloads + + def forward(self): + workloads = Workload() + if self.tp_size > 1 or self.ep_size > 1: + workloads.append(LogItem( + comm_type=CommType.all_gather, + comm_group=CommGroup.ep_group, + msg_size=2 * self.ep_size * self.num_experts * self.tp_size, + stage=f"forward.MoE.preprocess", + )) + workloads.extend(self.permutation(stage="forward")) + workloads.extend(self.unpermutation(stage="forward")) + assert all([isinstance(workload, LogItem) for workload in workloads.workload]) + return workloads + + def backward(self): + workloads = Workload() + workloads.extend(self.permutation(stage="backward")) + workloads.extend(self.unpermutation(stage="backward")) + assert all([isinstance(workload, LogItem) for workload in workloads.workload]) + return workloads + + +class SequentialMLP(MockedModel): + def __init__(self): + print("Not implement yet!") + pass + + +class MegatronTransformorLayer(MockedModel): + def __init__( + self, + hidden_size, + intermediate_size, + tp, + seq_len, + batch_size, + num_attention_heads, + layer_id, + expert_model_parallel_size, + moe_router_topk, + num_experts, + moe_grouped_gemm=True, + sequence_parallel_enabled=True, + computation_enable=False, + add_bias_linear=False, + moe_enable=False, + swiglu=False, + moe_intermediate_size=None, + ): + self.attention = MegatronAttention( + num_attention_heads, + hidden_size, + tp, + seq_len, + batch_size, + layer_id, + sequence_parallel_enabled, + computation_enable, + add_bias_linear, + ) + self.pre_mlp_layernorm = FusedLayernorm(hidden_size) + self.post_attention_layernorm_bias = MockedParam((hidden_size, 1)) + if moe_enable: + self.mlp = MOEMLP( + batch_size, + hidden_size, + tp, + expert_model_parallel_size, + moe_intermediate_size or intermediate_size, + seq_len, + moe_router_topk, + num_experts, + layer_id, + ) + else: + self.mlp = MegatronMlp( + hidden_size, + intermediate_size, + tp, + seq_len, + batch_size, + layer_id, + sequence_parallel_enabled, + computation_enable, + add_bias_linear, + swiglu=swiglu, + ) + + def forward(self): + workloads = Workload() + workloads.extend(self.attention.forward()) + workloads.extend(self.mlp.forward()) + assert all([isinstance(workload, LogItem) for workload in workloads.workload]) + return workloads + + def backward(self): + workloads = Workload() + workloads.extend(self.attention.backward()) + workloads.extend(self.mlp.backward()) + assert all([isinstance(workload, LogItem) for workload in workloads.workload]) + return workloads + + +class MegatronEmbedding(MockedModel): + def __init__(self, padded_vocab_size, hidden_size, tp, seq_len, batch_size): + self.name = "embedding_layer" + self.layer_id = 0 + num_embedding_per_partition = divide(padded_vocab_size, tp) + self.word_embedding = MockedParam( + (4 * num_embedding_per_partition, hidden_size), name=self.name + ) + self.tensor_model_parallel_size = tp + # TODO : position embedding shape is max_sequence_length not sequence_length + self.position_embedding = MockedParam((seq_len, hidden_size)) + self.comm_size = 2 * batch_size * seq_len * hidden_size + + def forward(self): + workloads = Workload() + if self.tensor_model_parallel_size > 1: + workloads.append( + LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.tp_group, + comm_group_size=self.tensor_model_parallel_size, + msg_size=self.comm_size, + stage="forward.MegatronEmbedding", + ) + ) + return workloads + + def backward(self): + workloads = Workload() + if self.tensor_model_parallel_size > 1: + workloads.append( + LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.tp_group, + comm_group_size=self.tensor_model_parallel_size, + msg_size=self.comm_size, + stage="backward.MegatronEmbedding", + ) + ) + return workloads + + +class MegatronModel(MockedModel): + def __init__(self, config): + intermediate_size = getattr(config, "intermediate_size", getattr(config, "ffn_hidden_size", 4 * config.hidden_size)) + swiglu = getattr(config, "swiglu", False) + moe_intermediate_size = getattr(config, "moe_intermediate_size", None) + + self.embedding = MegatronEmbedding( + config.padded_vocab_size, + config.hidden_size, + config.tensor_model_parallel_size, + config.seq_length, + config.micro_batch, + ) + self.layers = [ + MegatronTransformorLayer( + config.hidden_size, + intermediate_size, + config.tensor_model_parallel_size, + config.seq_length, + config.micro_batch, + config.num_attention_heads, + i, + config.expert_model_parallel_size, + config.moe_router_topk, + config.num_experts, + config.moe_grouped_gemm, + config.enable_sequence_parallel, + config.computation_enable, + config.add_bias_linear, + config.moe_enable, + swiglu=swiglu, + moe_intermediate_size=moe_intermediate_size, + ) + for i in range(config.num_layers) + ] + self.final_norm = MegatronColumnLinear( + config.hidden_size, + config.padded_vocab_size, + config.tensor_model_parallel_size, + config.seq_length, + config.micro_batch, + 1, + "final", + sequence_parallel_enabled=config.enable_sequence_parallel, + computation_enable=config.computation_enable, + add_bias_linear=config.add_bias_linear, + ) + + def forward(self): + workloads = Workload() + workloads.extend(self.embedding.forward()) + for layer in self.layers: + workloads.extend(layer.forward()) + assert all([isinstance(workload, LogItem) for workload in workloads.workload]) + return workloads + + def backward(self): + workloads = Workload() + for layer in self.layers[::-1]: + workloads.extend(layer.backward()) + workloads.extend(self.embedding.backward()) + assert all([isinstance(workload, LogItem) for workload in workloads.workload]) + return workloads diff --git a/aicb/workload_generator/mocked_model/training/MockedQwen3.py b/aicb/workload_generator/mocked_model/training/MockedQwen3.py new file mode 100644 index 000000000..76ecc7dab --- /dev/null +++ b/aicb/workload_generator/mocked_model/training/MockedQwen3.py @@ -0,0 +1,461 @@ +""" +Mocked Qwen3 dense model for AICB training workload generation. + +Based on MockedMegatron.py. Qwen3-specific changes versus LLAMA / GPT-3: + + * GQA – separate num_key_value_heads (Megatron hardcodes MHA) + * head_dim from config (128) instead of hidden_size // num_attention_heads + * QK-Norm – compute-only RMSNorm on query / key (hardcoded, always on) + * MLP fix – SwiGLU down-projection uses intermediate_size, not 2*intermediate + +Imports MegatronColumnLinear, MegatronRowLinear, MOEMLP +from MockedMegatron so TP communication primitives are NOT duplicated. +""" + +from utils.utils import divide, CommType, CommGroup +from workload_generator.mocked_model.MockedModel import MockedModel, MockedParam, MockedParamsBase +from log_analyzer.log import Workload, LogItem + +# --------------------------------------------------------------------------- +# IMPORT: reuse Megatron's TP communication primitives unchanged +# --------------------------------------------------------------------------- +from workload_generator.mocked_model.training.MockedMegatron import ( + MegatronColumnLinear, + MegatronRowLinear, + MOEMLP, +) + + +# =================================================================== +# Qwen3-specific modules (everything below is new) +# =================================================================== + +class Qwen3RMSNorm(MockedModel): + """RMSNorm weight parameter. Used for layernorm and QK-Norm.""" + + def __init__(self, dim, prefix_name="norm", layer_id=0): + self.layer_id = layer_id + self.name = prefix_name + self.weight = MockedParam((dim, 1), name=prefix_name) + + +# --------------------------------------------------------------------------- +# Embedding (no Megatron artifacts: no 4x vocab multiplier, no learned pos) +# --------------------------------------------------------------------------- + +class Qwen3Embedding(MockedModel): + """Qwen3 embedding layer. Qwen3 uses RoPE, not learned position embeddings.""" + + def __init__(self, vocab_size, hidden_size, tp, seq_len, batch_size): + self.name = "embedding_layer" + self.layer_id = 0 + num_embedding_per_partition = divide(vocab_size, tp) + self.word_embedding = MockedParam( + (num_embedding_per_partition, hidden_size), name="word_embedding" + ) + self.tensor_model_parallel_size = tp + self.comm_size = 2 * batch_size * seq_len * hidden_size + + def forward(self): + w = Workload() + if self.tensor_model_parallel_size > 1: + w.append(LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.tp_group, + comm_group_size=self.tensor_model_parallel_size, + msg_size=self.comm_size, + stage="forward.Qwen3Embedding", + )) + return w + + def backward(self): + w = Workload() + if self.tensor_model_parallel_size > 1: + w.append(LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.tp_group, + comm_group_size=self.tensor_model_parallel_size, + msg_size=self.comm_size, + stage="backward.Qwen3Embedding", + )) + return w + + +# --------------------------------------------------------------------------- +# Attention (GQA + hardcoded QK-Norm) +# --------------------------------------------------------------------------- + +class Qwen3Attention(MockedModel): + """ + Qwen3 attention with Grouped Query Attention and QK-Norm. + + Projection sizes (GQA-aware): + q_proj: hidden_size -> num_attention_heads * head_dim + k_proj: hidden_size -> num_key_value_heads * head_dim + v_proj: hidden_size -> num_key_value_heads * head_dim + o_proj: num_attention_heads * head_dim -> hidden_size + + The three input projections are fused into a single MegatronColumnLinear + so the ALLGATHER on the (unchanged) hidden_size input happens once. + + QK-Norm (hardcoded, always on): + - two RMSNorm(head_dim) applied per-head after projection, before RoPE + - compute-only – zero communication impact + - 256 floats per layer → negligible parameter-count impact + """ + + def __init__( + self, + hidden_size, + num_attention_heads, + num_key_value_heads, + head_dim, + tp, + seq_len, + batch_size, + layer_id, + sequence_parallel_enabled=True, + computation_enable=False, + add_bias_linear=False, + ): + self.name = "attention_layer" + self.layer_id = layer_id + + self.query_projection_size = num_attention_heads * head_dim + self.kv_projection_size = num_key_value_heads * head_dim + + # Fused QKV column – 1 ALLGATHER on hidden_size, same comm vol as Megatron + self.qkv = MegatronColumnLinear( + hidden_size, + self.query_projection_size + 2 * self.kv_projection_size, + tp, seq_len, batch_size, + layer_id, "attention", + sequence_parallel_enabled, computation_enable, + name="attention_qkv_column", + add_bias_linear=add_bias_linear, + ) + + # QK-Norm – compute-only, no collectives + self.q_norm = Qwen3RMSNorm(head_dim, "attention_q_norm", layer_id) + self.k_norm = Qwen3RMSNorm(head_dim, "attention_k_norm", layer_id) + + # Output projection – REDUCESCATTER on hidden_size, same as Megatron + self.attention_dense = MegatronRowLinear( + self.query_projection_size, hidden_size, + tp, seq_len, batch_size, + layer_id, "attention", + sequence_parallel_enabled, computation_enable, + name="attention_row", + add_bias_linear=add_bias_linear, + ) + + def forward(self): + w = Workload() + w.extend(self.qkv.forward()) + # qk_norm.forward() is intentionally NOT called — pure compute, no comms + w.extend(self.attention_dense.forward()) + return w + + def backward(self): + w = Workload() + w.extend(self.qkv.backward()) + w.extend(self.attention_dense.backward()) + return w + + +# --------------------------------------------------------------------------- +# SwiGLU MLP (fixes the MegatronMLP sizing issue) +# --------------------------------------------------------------------------- + +class Qwen3Mlp(MockedModel): + """ + Qwen3 SwiGLU MLP with CORRECT projection sizing. + + Gate + Up (fused column): hidden_size -> 2 * intermediate_size + Down (row): intermediate_size -> hidden_size + + NOTE: MegatronMlp uses the SAME `ffn_hidden_size` for both column and row. + For SwiGLU that overcounts down-projection params by 2×. This class fixes it. + """ + + def __init__( + self, + hidden_size, + intermediate_size, + tp, seq_len, batch_size, layer_id, + sequence_parallel_enabled=True, + computation_enable=False, + add_bias_linear=False, + ): + self.name = "mlp_layer" + self.layer_id = layer_id + + # gate_proj + up_proj fused (column – ALLGATHER on hidden_size) + self.gate_up = MegatronColumnLinear( + hidden_size, + 2 * intermediate_size, + tp, seq_len, batch_size, + layer_id, "mlp", + sequence_parallel_enabled, computation_enable, + name="mlp_gate_up_column", + add_bias_linear=add_bias_linear, + ) + + # down_proj (row – REDUCESCATTER on hidden_size) + # input is intermediate_size, NOT 2*intermediate (the fix) + self.down = MegatronRowLinear( + intermediate_size, + hidden_size, + tp, seq_len, batch_size, + layer_id, "mlp", + sequence_parallel_enabled, computation_enable, + name="mlp_down_row", + add_bias_linear=add_bias_linear, + ) + + def forward(self): + w = Workload() + w.extend(self.gate_up.forward()) + w.extend(self.down.forward()) + return w + + def backward(self): + w = Workload() + w.extend(self.gate_up.backward()) + w.extend(self.down.backward()) + return w + + +# --------------------------------------------------------------------------- +# Transformer Layer (pre-norm, same structure as LLaMA) +# --------------------------------------------------------------------------- + +class Qwen3TransformerLayer(MockedModel): + """ + input -> input_layernorm -> attention (+ residual) + -> post_attention_layernorm -> mlp (+ residual) + + Supports both dense (Qwen3Mlp) and MoE (MOEMLP) FFN via moe_enable. + """ + + def __init__( + self, + hidden_size, + intermediate_size, + num_attention_heads, + num_key_value_heads, + head_dim, + tp, seq_len, batch_size, layer_id, + # MoE params (ignored when moe_enable=False) + moe_enable=False, + moe_intermediate_size=0, + moe_router_topk=0, + num_experts=0, + expert_model_parallel_size=1, + # Standard params + sequence_parallel_enabled=True, + computation_enable=False, + add_bias_linear=False, + ): + self.name = "transformer_layer" + self.layer_id = layer_id + + self.input_layernorm = Qwen3RMSNorm(hidden_size, "input_layernorm", layer_id) + self.post_attention_layernorm = Qwen3RMSNorm(hidden_size, "post_attn_norm", layer_id) + + self.attention = Qwen3Attention( + hidden_size, num_attention_heads, num_key_value_heads, head_dim, + tp, seq_len, batch_size, layer_id, + sequence_parallel_enabled, computation_enable, add_bias_linear, + ) + + if moe_enable: + self.mlp = MOEMLP( + batch_size, hidden_size, tp, expert_model_parallel_size, + moe_intermediate_size, seq_len, moe_router_topk, + num_experts, layer_id, + ) + else: + self.mlp = Qwen3Mlp( + hidden_size, intermediate_size, + tp, seq_len, batch_size, layer_id, + sequence_parallel_enabled, computation_enable, add_bias_linear, + ) + + def forward(self): + w = Workload() + w.extend(self.attention.forward()) + w.extend(self.mlp.forward()) + return w + + def backward(self): + w = Workload() + w.extend(self.attention.backward()) + w.extend(self.mlp.backward()) + return w + + +# --------------------------------------------------------------------------- +# Full model +# --------------------------------------------------------------------------- + +class Qwen3Model(MockedModel): + """ + Complete Qwen3 dense model. + + Config attributes consumed (from HF config.json): + hidden_size, intermediate_size, num_hidden_layers, + num_attention_heads, num_key_value_heads, head_dim, vocab_size + + Training attributes (from CLI / args): + tensor_model_parallel_size, seq_length, micro_batch, + enable_sequence_parallel, computation_enable, add_bias_linear + """ + + def __init__(self, config): + h = config.hidden_size + ff = config.intermediate_size + n_heads = config.num_attention_heads + n_kv = config.num_key_value_heads + h_dim = getattr(config, "head_dim", h // n_heads) + n_layers = config.num_hidden_layers + V = config.vocab_size + tp = config.tensor_model_parallel_size + S = config.seq_length + B = config.micro_batch + sp = getattr(config, "enable_sequence_parallel", True) + comp = getattr(config, "computation_enable", False) + bias = getattr(config, "add_bias_linear", False) + + # MoE config (ignored when moe_enable=False) + moe = getattr(config, "moe_enable", False) + moe_ff = getattr(config, "moe_intermediate_size", 0) + moe_topk = getattr(config, "moe_router_topk", 0) + moe_n_exp = getattr(config, "num_experts", 0) + moe_ep = getattr(config, "expert_model_parallel_size", 1) + + tie_emb = getattr(config, "tie_word_embeddings", False) + + self.embedding = Qwen3Embedding(V, h, tp, S, B) + + self.layers = [ + Qwen3TransformerLayer( + h, ff, n_heads, n_kv, h_dim, + tp, S, B, i, + moe_enable=moe, + moe_intermediate_size=moe_ff, + moe_router_topk=moe_topk, + num_experts=moe_n_exp, + expert_model_parallel_size=moe_ep, + sequence_parallel_enabled=sp, + computation_enable=comp, + add_bias_linear=bias, + ) + for i in range(n_layers) + ] + + self.final_norm = Qwen3RMSNorm(h, "final_norm", n_layers + 1) + + self.lm_head = MegatronColumnLinear( + h, V, tp, S, B, n_layers + 2, "lm_head", + sequence_parallel_enabled=sp, + computation_enable=comp, + add_bias_linear=bias, + ) + + # When tie_word_embeddings=True the lm_head shares weights with the + # embedding table. Communication is unchanged (the ColumnLinear still + # runs), but we must not double-count the weight in parameter totals. + if tie_emb: + self.lm_head.weight = MockedParam((0,), name="lm_head_tied") + + def forward(self): + w = Workload() + w.extend(self.embedding.forward()) + for layer in self.layers: + w.extend(layer.forward()) + w.extend(self.lm_head.forward()) + return w + + def backward(self): + w = Workload() + w.extend(self.lm_head.backward()) + for layer in reversed(self.layers): + w.extend(layer.backward()) + w.extend(self.embedding.backward()) + return w + + +# =================================================================== +# Config / params class +# =================================================================== + +class Qwen3Params(MockedParamsBase): + """ + Qwen3 training configuration. + + Loads HF config.json via load_from_config (inherited). + Overrides via command-line args via load_from_args (inherited). + + Default values are for Qwen3-8B. + """ + + def __init__(self, config_file=None, args=None): + # --- HF config.json defaults (Qwen3-8B) --- + self.hidden_size = 4096 + self.intermediate_size = 12288 + self.num_hidden_layers = 36 + self.num_attention_heads = 32 + self.num_key_value_heads = 8 + self.head_dim = 128 + self.vocab_size = 151936 + self.rms_norm_eps = 1e-6 + self.rope_theta = 1_000_000 + self.max_position_embeddings = 40960 + self.tie_word_embeddings = False + self.attention_bias = False + + # --- Training defaults --- + self.tensor_model_parallel_size = 1 + self.expert_model_parallel_size = 1 + self.pipeline_model_parallel = 1 + self.world_size = 1 + self.seq_length = 4096 + self.micro_batch = 1 + self.computation_enable = False + self.enable_sequence_parallel = True + self.add_bias_linear = False + self.moe_enable = False + self.moe_intermediate_size = 0 + self.moe_router_topk = 0 + self.num_experts = 0 + + super().__init__("Qwen3", "Qwen3", config_file, args) + + +# =================================================================== +# Smoke test +# =================================================================== + +if __name__ == "__main__": + import sys, os + + config_file = sys.argv[1] if len(sys.argv) > 1 else None + if config_file and not os.path.exists(config_file): + print(f"[ERROR] config file not found: {config_file}") + sys.exit(1) + + cfg = Qwen3Params(config_file) + model = Qwen3Model(cfg) + + total_params = sum(p.numel() for p in model.parameters()) + total_bytes = sum(p.msg_size() for p in model.parameters()) + + print(f"Qwen3 | {cfg.hidden_size=} {cfg.intermediate_size=}") + print(f" | {cfg.num_hidden_layers=} {cfg.num_attention_heads=} {cfg.num_key_value_heads=} {cfg.head_dim=}") + print(f" | {cfg.vocab_size=} tp={cfg.tensor_model_parallel_size} seq={cfg.seq_length} mbs={cfg.micro_batch}") + print(f" | total params: {total_params:,} ({total_bytes/1e9:.2f} GB BF16)") + + fwd = model.forward() + bwd = model.backward() + print(f" | fwd workloads: {len(fwd.workload)} bwd workloads: {len(bwd.workload)}") diff --git a/aicb/workload_generator/mocked_model/training/MockedQwen3_5.py b/aicb/workload_generator/mocked_model/training/MockedQwen3_5.py new file mode 100644 index 000000000..af38c4ccc --- /dev/null +++ b/aicb/workload_generator/mocked_model/training/MockedQwen3_5.py @@ -0,0 +1,823 @@ +""" +Copyright (c) 2024, Alibaba Group; +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Mocked model for Qwen3.5 dense + MoE (training workloads). + +Architecture: + - 3:1 hybrid attention: 75% GatedDeltaNet (linear O(L) + causal conv1d), + 25% full GQA attention (head_dim=256, partial_rotary=0.25, MRoPE, attn_output_gate). + - MoE: 256-512 experts, top-8 or top-10 routing, WITH shared experts + (routed experts + 1 always-active shared expert). + - Pre-LN RMSNorm, SwiGLU MLP. Full-attention layers use QK-Norm (per-head + RMSNorm on Q/K). GatedDeltaNet layers use RMSNormGated. head_dim=256, + rope_theta=10M. QK-Norm = pure local compute, no communication impact. + - Config keys: hidden_size, intermediate_size, full_attention_interval, + layer_types, linear_key_head_dim, linear_num_key_heads, etc. + +Key simulation notes: + - GatedDeltaNet layers: ZERO communication on attention (all local + state-matrix recurrence + causal conv1d). Only MLP/MoE adds comm. + - Full-attention layers: QKV projection + output projection, both TP-sharded + (same communication pattern as Qwen3/Megatron, larger dims). + - Shared expert: additional dense MLP (MegatronMlp pattern), always active. +""" + +from utils.utils import divide, CommType, CommGroup +from workload_generator.mocked_model.MockedModel import MockedModel, Linear, MockedParam, MockedParamsBase +from log_analyzer.log import Workload, LogItem + + +# --------------------------------------------------------------------------- +# Tensor-parallel building blocks (identically-ported from MockedQwen3.py) +# --------------------------------------------------------------------------- + +class Qwen3_5RowLinear(MockedModel): + """Row-parallel linear (output dimension sharded).""" + + def __init__( + self, input_size, output_size, tp, seq_len, batch_size, layer_id, + prefix_name, sequence_parallel_enabled=True, computation_enable=False, + name=None, add_bias_linear=False, + ): + self.layer_id = layer_id + self.name = prefix_name + "_row" + self.input_size, self.output_size = input_size, output_size + self.input_size_per_partition = divide(input_size, tp) + self.weight = MockedParam((output_size, self.input_size_per_partition), name=name) + if add_bias_linear: + self.bias = MockedParam((output_size, 1), name=self.name + "_bias") + self.sequence_parallel_enabled = sequence_parallel_enabled + self.computation_enable = computation_enable + self.tensor_model_parallel_size, self.seq_len, self.batch_size = tp, seq_len, batch_size + self.comm_size = 2 * seq_len * batch_size * output_size + + def activation_memory(self): + return self.seq_len * self.input_size + + def forward(self): + workloads = Workload() + if self.computation_enable: + workloads.append(LogItem( + comm_type=CommType.computation, + msg_size=((self.seq_len, self.batch_size, self.input_size_per_partition), + (self.input_size_per_partition, self.output_size)), + stage="forward.Qwen3_5RowLinear." + self.name)) + if self.tensor_model_parallel_size > 1: + if self.sequence_parallel_enabled: + workloads.append(LogItem( + comm_type=CommType.reduce_scatter, + comm_group=CommGroup.tp_group, + comm_group_size=self.tensor_model_parallel_size, + msg_size=self.comm_size, + stage="forward.Qwen3_5RowLinear")) + else: + workloads.append(LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.tp_group, + comm_group_size=self.tensor_model_parallel_size, + msg_size=self.comm_size, + stage="forward.Qwen3_5RowLinear")) + return workloads + + def backward(self): + workloads = Workload() + if self.tensor_model_parallel_size > 1: + if self.sequence_parallel_enabled: + workloads.append(LogItem( + comm_type=CommType.all_gather, + comm_group=CommGroup.tp_group, + comm_group_size=self.tensor_model_parallel_size, + msg_size=self.comm_size, + stage="backward.Qwen3_5RowLinear")) + if self.computation_enable: + workloads.append(LogItem( + comm_type=CommType.computation, + msg_size=((self.seq_len, self.batch_size, self.output_size), self.weight.shape), + stage="backward.Qwen3_5RowLinear." + self.name)) + workloads.append(LogItem( + comm_type=CommType.computation, + msg_size=((self.output_size, self.seq_len * self.batch_size), + (self.seq_len * self.batch_size, self.input_size_per_partition)), + stage="backward.Qwen3_5RowLinear." + self.name)) + return workloads + + +class Qwen3_5ColumnLinear(MockedModel): + """Column-parallel linear (input dimension sharded).""" + + def __init__( + self, input_size, output_size, tp, seq_len, batch_size, layer_id, + prefix_name, sequence_parallel_enabled=True, computation_enable=False, + name=None, add_bias_linear=False, + ): + self.layer_id = layer_id + self.name = prefix_name + "_column" + self.input_size, self.output_size = input_size, output_size + self.output_size_per_partition = divide(output_size, tp) + self.weight = MockedParam((input_size, self.output_size_per_partition), name=name) + if add_bias_linear: + self.bias = MockedParam((self.output_size_per_partition, 1), name=self.name + "_bias") + self.sequence_parallel_enabled = sequence_parallel_enabled + self.computation_enable = computation_enable + self.tensor_model_parallel_size, self.seq_len, self.batch_size = tp, seq_len, batch_size + self.comm_size = 2 * seq_len * batch_size * input_size + if self.tensor_model_parallel_size > 1 and self.sequence_parallel_enabled: + self.seq_len *= self.tensor_model_parallel_size + + def activation_memory(self): + return self.seq_len * self.input_size + + def forward(self): + workloads = Workload() + if self.tensor_model_parallel_size > 1: + if self.sequence_parallel_enabled: + workloads.append(LogItem( + comm_type=CommType.all_gather, + comm_group=CommGroup.tp_group, + comm_group_size=self.tensor_model_parallel_size, + msg_size=self.comm_size, + stage="forward.Qwen3_5ColumnLinear")) + if self.computation_enable: + workloads.append(LogItem( + comm_type=CommType.computation, + msg_size=((self.seq_len, self.batch_size, self.input_size), + (self.input_size, self.output_size_per_partition)), + stage="forward.Qwen3_5ColumnLinear." + self.name)) + return workloads + + def backward(self): + workloads = Workload() + if self.tensor_model_parallel_size > 1: + if self.sequence_parallel_enabled: + workloads.append(LogItem( + comm_type=CommType.all_gather, + comm_group=CommGroup.tp_group, + comm_group_size=self.tensor_model_parallel_size, + msg_size=self.comm_size, + stage="backward.Qwen3_5ColumnLinear")) + if self.computation_enable: + workloads.append(LogItem( + comm_type=CommType.computation, + msg_size=((self.seq_len, self.batch_size, self.output_size_per_partition), + (self.output_size_per_partition, self.input_size)), + stage="backward.Qwen3_5ColumnLinear." + self.name)) + if self.tensor_model_parallel_size > 1: + if self.sequence_parallel_enabled: + workloads.append(LogItem( + comm_type=CommType.reduce_scatter, + comm_group=CommGroup.tp_group, + comm_group_size=self.tensor_model_parallel_size, + msg_size=self.comm_size, + stage="backward.Qwen3_5ColumnLinear")) + if self.computation_enable: + workloads.append(LogItem( + comm_type=CommType.computation, + msg_size=((self.output_size_per_partition, self.seq_len * self.batch_size), + (self.seq_len * self.batch_size, self.input_size)), + stage="backward.Qwen3_5ColumnLinear." + self.name)) + if self.tensor_model_parallel_size > 1: + if not self.sequence_parallel_enabled: + workloads.append(LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.tp_group, + comm_group_size=self.tensor_model_parallel_size, + msg_size=self.comm_size, + stage="backward.Qwen3_5ColumnLinear")) + return workloads + + +# --------------------------------------------------------------------------- +# Qwen3.5-specific modules +# --------------------------------------------------------------------------- + +class Qwen3_5RMSNorm(MockedModel): + """RMSNorm with weight parameter.""" + + def __init__(self, dim, prefix_name, layer_id=0): + self.layer_id = layer_id + self.name = prefix_name + self.weight = MockedParam((dim, 1), name=prefix_name) + + +class Qwen3_5FullAttention(MockedModel): + """ + Standard GQA full attention: every 4th layer. + + head_dim=256, partial_rotary=0.25, attn_output_gate=True, MRoPE. + Communication pattern: same as MegatronAttention -- TP all-gather on QKV + column input, TP reduce-scatter on attention output. + """ + + def __init__( + self, + hidden_size, + num_attention_heads, + num_key_value_heads, + head_dim, + tp, + seq_len, + batch_size, + layer_id, + sequence_parallel_enabled=True, + computation_enable=False, + add_bias_linear=False, + ): + self.name = "full_attention_layer" + self.layer_id = layer_id + self.num_attention_heads = num_attention_heads + self.num_key_value_heads = num_key_value_heads + self.head_dim = head_dim + self.query_projection_size = num_attention_heads * head_dim + self.kv_projection_size = num_key_value_heads * head_dim + + # QKV projection: hidden_size -> Q + K + V (combined) + total_qkv_dim = self.query_projection_size + 2 * self.kv_projection_size + self.qkv = Qwen3_5ColumnLinear( + hidden_size, total_qkv_dim, tp, seq_len, batch_size, layer_id, + "full_attention", + sequence_parallel_enabled, computation_enable, + name="qkv_column", add_bias_linear=add_bias_linear, + ) + + # Output projection: heads * head_dim -> hidden_size + self.attention_dense = Qwen3_5RowLinear( + self.query_projection_size, hidden_size, tp, seq_len, batch_size, + layer_id, "full_attention", + sequence_parallel_enabled, computation_enable, + name="attention_row", add_bias_linear=add_bias_linear, + ) + + def forward(self): + workloads = Workload() + workloads.extend(self.qkv.forward()) + # MRoPE, partial_rotary, attn_output_gate happen here: all local compute + workloads.extend(self.attention_dense.forward()) + return workloads + + def backward(self): + workloads = Workload() + workloads.extend(self.qkv.backward()) + workloads.extend(self.attention_dense.backward()) + return workloads + + +class Qwen3_5GatedDeltaNet(MockedModel): + """ + Linear attention (O(L) complexity): 75% of layers. + + State recurrence: S_t = S_{t-1} * alpha * (I - beta * k * k^T) + beta * v * k^T + with causal_conv1d(kernel_dim=4) for local positional info. + + Communication model (comm-accurate): + - QKVZ + BA input projection: replicated across TP ranks, local compute + - Causal conv1d: local, no communication + - Gated delta rule recurrence: local, no communication + - Output projection: replicated across TP ranks, local compute + + All operations in GatedDeltaNet are local compute -- the projections are + replicated (not TP-sharded) because their parameter count is small relative + to the MLP/MoE that follows, and the recurrent state makes TP complex. + forward()/backward() return empty Workload. + + Raw MockedParam objects track full-size parameters for accurate DP gradient + sync sizing in the step() method. + """ + + def __init__( + self, + hidden_size, + linear_key_head_dim, + linear_value_head_dim, + linear_num_key_heads, + linear_num_value_heads, + linear_conv_kernel_dim, + layer_id, + ): + self.name = "gated_deltanet_layer" + self.layer_id = layer_id + + # QKVZ combined projection: + # Q = linear_num_key_heads * linear_key_head_dim + # K = linear_num_key_heads * linear_key_head_dim + # V = linear_num_value_heads * linear_value_head_dim + # Z = linear_num_value_heads * linear_value_head_dim (output gate) + q_dim = linear_num_key_heads * linear_key_head_dim + v_dim = linear_num_value_heads * linear_value_head_dim + qkvz_dim = q_dim * 2 + v_dim * 2 + + # BA projection: beta gate + decay log-delta per value head + ba_dim = linear_num_value_heads * 2 + + # Combined input projection weight: full-size, replicated (no TP) + self.in_proj_weight = MockedParam( + (hidden_size, qkvz_dim + ba_dim), name="gdn_in_proj" + ) + + # Causal conv1d weight: small parameter, local + self.conv_weight = MockedParam( + (linear_num_key_heads, 1, linear_conv_kernel_dim), name="gdn_conv1d" + ) + + # Output projection weight: full-size, replicated (no TP) + self.out_proj_weight = MockedParam( + (v_dim, hidden_size), name="gdn_out_proj" + ) + + def forward(self): + """All local compute -- no TP collectives generated.""" + return Workload() + + def backward(self): + """All local compute -- no TP collectives generated.""" + return Workload() + + +class Qwen3_5Mlp(MockedModel): + """ + Qwen3.5 SwiGLU MLP for dense models. + + Uses intermediate_size from config. + gate_proj + up_proj (combined ColumnLinear) -> down_proj (RowLinear). + """ + + def __init__( + self, hidden_size, intermediate_size, tp, seq_len, batch_size, layer_id, + sequence_parallel_enabled=True, computation_enable=False, add_bias_linear=False, + ): + self.name = "mlp_layer" + self.layer_id = layer_id + # SwiGLU: gate_proj and up_proj combined -> 2 * intermediate_size + self.gate_up_proj = Qwen3_5ColumnLinear( + hidden_size, 2 * intermediate_size, tp, seq_len, batch_size, layer_id, + "mlp", sequence_parallel_enabled, computation_enable, + name="mlp_gate_up_column", add_bias_linear=add_bias_linear, + ) + self.down_proj = Qwen3_5RowLinear( + intermediate_size, hidden_size, tp, seq_len, batch_size, layer_id, + "mlp", sequence_parallel_enabled, computation_enable, + name="mlp_down_row", add_bias_linear=add_bias_linear, + ) + + def forward(self): + workloads = Workload() + workloads.extend(self.gate_up_proj.forward()) + workloads.extend(self.down_proj.forward()) + return workloads + + def backward(self): + workloads = Workload() + workloads.extend(self.gate_up_proj.backward()) + workloads.extend(self.down_proj.backward()) + return workloads + + +class Qwen3_5MoEMLP(MockedModel): + """ + Qwen3.5 MoE layer with shared experts. + + Standard Megatron MoE pattern: + - All-to-all dispatch/combine across EP for routed experts. + - TP all-gather / reduce-scatter within MoE computation. + - Shared expert: independent dense SwiGLU MLP, always active, no routing. + + Qwen3.5 MoE specifics: + - 256 experts (122B-A10B, 35B-A3B) or 512 experts (397B-A17B) + - top-8 or top-10 routing + - moe_intermediate_size per expert + - 1 shared expert with shared_expert_intermediate_size + """ + + def __init__( + self, + batch_size, + hidden_size, + tp, + expert_model_parallel_size, + moe_intermediate_size, + seq_len, + topk, + num_experts, + shared_expert_intermediate_size, + layer_id, + sequence_parallel_enabled=True, + computation_enable=False, + add_bias_linear=False, + ): + self.name = "moe_layer" + self.layer_id = layer_id + self.tp_size = tp + self.ep_size = expert_model_parallel_size + self.topk = topk + self.seq_len = seq_len + self.num_experts = num_experts + self.batch_size = batch_size + self.hidden_size = hidden_size + + # Routed experts: weights concatenated + TP-sharded + num_local_experts = num_experts // expert_model_parallel_size + fc1_output_size = moe_intermediate_size * num_local_experts + fc1_output_size_per_partition = divide(fc1_output_size, tp) + fc2_input_size = moe_intermediate_size * num_local_experts + fc2_input_size_per_partition = divide(fc2_input_size, tp) + self.weight1 = MockedParam((hidden_size, fc1_output_size_per_partition), name="moe_gate_up") + self.weight2 = MockedParam((fc2_input_size_per_partition, hidden_size), name="moe_down") + + # Shared expert: dense SwiGLU MLP, always active + if shared_expert_intermediate_size > 0: + self.shared_expert = Qwen3_5Mlp( + hidden_size, shared_expert_intermediate_size, tp, seq_len, + batch_size, layer_id, + sequence_parallel_enabled, computation_enable, add_bias_linear, + ) + else: + self.shared_expert = None + + def _dispatch_combine(self, stage): + """All-to-all token dispatch and combine across EP group.""" + workloads = Workload() + if self.ep_size > 1: + # Dispatch: send tokens to expert-owning EP ranks + # Dividing by ep_size: each EP rank receives ~1/ep of total tokens + workloads.append(LogItem( + comm_type=CommType.all_to_all, + comm_group=CommGroup.ep_group, + msg_size=(self.seq_len * self.hidden_size * self.batch_size * self.topk // self.tp_size // self.ep_size) * 2, + stage=f"{stage}.MoE.dispatch", + )) + if self.tp_size > 1: + # Within TP: gather full token batch for grouped GEMM + # Dividing by ep_size: after dispatch each EP rank holds ~1/ep of tokens + workloads.append(LogItem( + comm_type=CommType.all_gather, + comm_group=CommGroup.tp_group, + msg_size=2 * self.hidden_size * self.topk * self.batch_size * self.seq_len // self.ep_size, + stage=f"{stage}.MoE.permutation", + )) + return workloads + + def _combine_undispatch(self, stage): + """Reverse of dispatch: reduce-scatter TP then all-to-all EP.""" + workloads = Workload() + if self.tp_size > 1: + # Dividing by ep_size: after dispatch each EP rank holds ~1/ep of tokens + workloads.append(LogItem( + comm_type=CommType.reduce_scatter, + comm_group=CommGroup.tp_group, + msg_size=2 * self.hidden_size * self.batch_size * self.topk * self.seq_len // self.ep_size, + stage=f"{stage}.MoE.unpermutation", + )) + if self.ep_size > 1: + # Dividing by ep_size: each EP rank sends ~1/ep of total tokens + workloads.append(LogItem( + comm_type=CommType.all_to_all, + comm_group=CommGroup.ep_group, + msg_size=(self.seq_len * self.hidden_size * self.batch_size * self.topk // self.tp_size // self.ep_size) * 2, + stage=f"{stage}.MoE.combine", + )) + return workloads + + def forward(self): + workloads = Workload() + if self.shared_expert is not None: + workloads.extend(self.shared_expert.forward()) + if self.tp_size > 1 or self.ep_size > 1: + workloads.append(LogItem( + comm_type=CommType.all_gather, + comm_group=CommGroup.ep_group, + msg_size=2 * self.ep_size * self.num_experts * self.tp_size, + stage="forward.MoE.preprocess", + )) + workloads.extend(self._dispatch_combine(stage="forward")) + workloads.extend(self._combine_undispatch(stage="forward")) + return workloads + + def backward(self): + workloads = Workload() + if self.shared_expert is not None: + workloads.extend(self.shared_expert.backward()) + workloads.extend(self._dispatch_combine(stage="backward")) + workloads.extend(self._combine_undispatch(stage="backward")) + return workloads + + +class Qwen3_5TransformerLayer(MockedModel): + """ + Hybrid Qwen3.5 transformer layer. + + 75% GatedDeltaNet (linear attention) + 25% full GQA attention. + Layer type determined by full_attention_interval: every Nth layer + uses full attention, all others use GatedDeltaNet. + """ + + def __init__( + self, + hidden_size, + intermediate_size, + num_attention_heads, + num_key_value_heads, + head_dim, + full_attention_interval, + # GatedDeltaNet params + linear_key_head_dim, + linear_value_head_dim, + linear_num_key_heads, + linear_num_value_heads, + linear_conv_kernel_dim, + # MoE params (ignored when moe_enable=False) + moe_enable, + moe_intermediate_size, + moe_router_topk, + num_experts, + expert_model_parallel_size, + shared_expert_intermediate_size, + # Standard training params + tp, + seq_len, + batch_size, + layer_id, + sequence_parallel_enabled=True, + computation_enable=False, + add_bias_linear=False, + ): + self.name = "transformer_layer" + self.layer_id = layer_id + + self.input_layernorm = Qwen3_5RMSNorm( + hidden_size, prefix_name="input_layernorm", layer_id=layer_id + ) + self.post_attention_layernorm = Qwen3_5RMSNorm( + hidden_size, prefix_name="post_attention_layernorm", layer_id=layer_id + ) + + # Determine attention type: every full_attention_interval-th layer is full + is_full_attention = (layer_id + 1) % full_attention_interval == 0 + + if is_full_attention: + self.attention = Qwen3_5FullAttention( + hidden_size, num_attention_heads, num_key_value_heads, head_dim, + tp, seq_len, batch_size, layer_id, + sequence_parallel_enabled, computation_enable, add_bias_linear, + ) + else: + self.attention = Qwen3_5GatedDeltaNet( + hidden_size, linear_key_head_dim, linear_value_head_dim, + linear_num_key_heads, linear_num_value_heads, + linear_conv_kernel_dim, + layer_id, + ) + + # MLP or MoE + if moe_enable: + self.mlp = Qwen3_5MoEMLP( + batch_size, hidden_size, tp, expert_model_parallel_size, + moe_intermediate_size, seq_len, moe_router_topk, + num_experts, shared_expert_intermediate_size, layer_id, + sequence_parallel_enabled, computation_enable, add_bias_linear, + ) + else: + self.mlp = Qwen3_5Mlp( + hidden_size, intermediate_size, tp, seq_len, batch_size, layer_id, + sequence_parallel_enabled, computation_enable, add_bias_linear, + ) + + def forward(self): + workloads = Workload() + workloads.extend(self.attention.forward()) + workloads.extend(self.mlp.forward()) + return workloads + + def backward(self): + workloads = Workload() + workloads.extend(self.attention.backward()) + workloads.extend(self.mlp.backward()) + return workloads + + +class Qwen3_5Embedding(MockedModel): + """Qwen3.5 embedding layer with TP vocabulary sharding.""" + + def __init__(self, vocab_size, hidden_size, tp, seq_len, batch_size): + self.name = "embedding_layer" + self.layer_id = 0 + num_embedding_per_partition = divide(vocab_size, tp) + self.word_embedding = MockedParam( + (num_embedding_per_partition, hidden_size), name="word_embedding" + ) + self.tensor_model_parallel_size = tp + self.comm_size = 2 * batch_size * seq_len * hidden_size + + def forward(self): + workloads = Workload() + if self.tensor_model_parallel_size > 1: + workloads.append(LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.tp_group, + comm_group_size=self.tensor_model_parallel_size, + msg_size=self.comm_size, + stage="forward.Qwen3_5Embedding")) + return workloads + + def backward(self): + workloads = Workload() + if self.tensor_model_parallel_size > 1: + workloads.append(LogItem( + comm_type=CommType.all_reduce, + comm_group=CommGroup.tp_group, + comm_group_size=self.tensor_model_parallel_size, + msg_size=self.comm_size, + stage="backward.Qwen3_5Embedding")) + return workloads + + +class Qwen3_5Model(MockedModel): + """ + Complete Qwen3.5 model: embedding -> N hybrid transformer layers -> lm_head. + + Supports both dense and MoE variants. MoE params are ignored when + moe_enable=False. + """ + + def __init__(self, config): + hidden_size = config.hidden_size + intermediate_size = config.intermediate_size + num_attention_heads = config.num_attention_heads + num_key_value_heads = config.num_key_value_heads + head_dim = getattr(config, "head_dim", 256) + num_layers = config.num_hidden_layers + vocab_size = config.vocab_size + tp = config.tensor_model_parallel_size + seq_len = config.seq_length + batch_size = config.micro_batch + + # Qwen3.5-specific + full_attention_interval = getattr(config, "full_attention_interval", 4) + linear_key_head_dim = getattr(config, "linear_key_head_dim", 128) + linear_value_head_dim = getattr(config, "linear_value_head_dim", 128) + linear_num_key_heads = getattr(config, "linear_num_key_heads", 16) + linear_num_value_heads = getattr(config, "linear_num_value_heads", 16) + linear_conv_kernel_dim = getattr(config, "linear_conv_kernel_dim", 4) + + moe_enable = config.moe_enable + moe_intermediate_size = getattr(config, "moe_intermediate_size", 0) + moe_router_topk = config.moe_router_topk + num_experts = config.num_experts + expert_ep = config.expert_model_parallel_size + shared_inter = getattr(config, "shared_expert_intermediate_size", 0) + + self.embedding = Qwen3_5Embedding(vocab_size, hidden_size, tp, seq_len, batch_size) + + self.layers = [ + Qwen3_5TransformerLayer( + hidden_size, intermediate_size, + num_attention_heads, num_key_value_heads, head_dim, + full_attention_interval, + linear_key_head_dim, linear_value_head_dim, + linear_num_key_heads, linear_num_value_heads, + linear_conv_kernel_dim, + moe_enable, moe_intermediate_size, moe_router_topk, + num_experts, expert_ep, shared_inter, + tp, seq_len, batch_size, i, + sequence_parallel_enabled=config.enable_sequence_parallel, + computation_enable=config.computation_enable, + add_bias_linear=config.add_bias_linear, + ) + for i in range(num_layers) + ] + + self.lm_head = Qwen3_5ColumnLinear( + hidden_size, vocab_size, tp, seq_len, batch_size, + num_layers + 1, "lm_head", + sequence_parallel_enabled=config.enable_sequence_parallel, + computation_enable=config.computation_enable, + add_bias_linear=config.add_bias_linear, + ) + self.final_norm = Qwen3_5RMSNorm( + hidden_size, prefix_name="final_norm", layer_id=num_layers + 2 + ) + + def forward(self): + workloads = Workload() + workloads.extend(self.embedding.forward()) + for layer in self.layers: + workloads.extend(layer.forward()) + workloads.extend(self.lm_head.forward()) + return workloads + + def backward(self): + workloads = Workload() + workloads.extend(self.lm_head.backward()) + for layer in self.layers[::-1]: + workloads.extend(layer.backward()) + workloads.extend(self.embedding.backward()) + return workloads + + +# --------------------------------------------------------------------------- +# Config / params class +# --------------------------------------------------------------------------- + +class Qwen3_5Params(MockedParamsBase): + """ + Qwen3.5 training configuration. + + Accepts a HuggingFace config.json file and/or command-line args. + Handles the nested 'text_config' structure from Qwen3.5 HF configs. + + Defaults are for Qwen3.5-9B dense. + """ + + def __init__(self, config_file=None, args=None): + # Default to Qwen3.5-9B dense + self.hidden_size = 4096 + self.intermediate_size = 12288 + self.num_hidden_layers = 32 + self.num_attention_heads = 16 + self.num_key_value_heads = 4 + self.head_dim = 256 + self.vocab_size = 248320 + self.rms_norm_eps = 1e-6 + self.rope_theta = 10_000_000 + self.max_position_embeddings = 262144 + + # Qwen3.5-specific defaults + self.full_attention_interval = 4 + self.linear_key_head_dim = 128 + self.linear_value_head_dim = 128 + self.linear_num_key_heads = 16 + self.linear_num_value_heads = 16 + self.linear_conv_kernel_dim = 4 + + # Training-specific defaults + self.tensor_model_parallel_size = 1 + self.expert_model_parallel_size = 1 + self.pipeline_model_parallel = 1 + self.world_size = 1 + self.seq_length = 4096 + self.micro_batch = 1 + self.computation_enable = False + self.enable_sequence_parallel = True + self.add_bias_linear = False + self.moe_enable = False + self.moe_router_topk = 0 + self.num_experts = 0 + self.moe_intermediate_size = 0 + self.shared_expert_intermediate_size = 0 + self.moe_grouped_gemm = True + + super().__init__("Qwen3.5", "Qwen3.5", config_file, args) + + # Unpack nested text_config if present (HF Qwen3.5 multimodal format) + if hasattr(self, 'text_config'): + for key, value in self.text_config.items(): + setattr(self, key, value) + + +# --------------------------------------------------------------------------- +# Quick smoke test +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + import sys + config_file = None + if len(sys.argv) > 1: + config_file = sys.argv[1] + args = Qwen3_5Params(config_file) + model = Qwen3_5Model(args) + print(f"Model: {args.model_name}") + print(f" hidden_size={args.hidden_size}") + print(f" intermediate_size={args.intermediate_size}") + print(f" num_hidden_layers={args.num_hidden_layers}") + print(f" num_attention_heads={args.num_attention_heads}") + print(f" num_key_value_heads={args.num_key_value_heads}") + print(f" head_dim={args.head_dim}") + print(f" vocab_size={args.vocab_size}") + print(f" full_attention_interval={args.full_attention_interval}") + if args.moe_enable: + print(f" MoE: experts={args.num_experts} topk={args.moe_router_topk}") + print(f" moe_intermediate_size={args.moe_intermediate_size}") + print(f" shared_expert_intermediate_size={args.shared_expert_intermediate_size}") + print(f" tp={args.tensor_model_parallel_size}") + print(f" seq_len={args.seq_length}") + print(f" batch_size={args.micro_batch}") + print(f" total params: {sum(p.numel() for p in model.parameters()):,}") + print(f" total layers: {len(model.layers)}") + + # Count layer types + full_layers = sum(1 for i in range(args.num_hidden_layers) + if (i + 1) % args.full_attention_interval == 0) + linear_layers = args.num_hidden_layers - full_layers + print(f" layer breakdown: {full_layers} full-attention + {linear_layers} GatedDeltaNet") + + fwd = model.forward() + print(f" forward workloads: {len(fwd.workload)}") + bwd = model.backward() + print(f" backward workloads: {len(bwd.workload)}") diff --git a/aicb/workload_generator/mocked_model/training/__init__.py b/aicb/workload_generator/mocked_model/training/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/aicb/workload_generator/workload_generator.py b/aicb/workload_generator/workload_generator.py new file mode 100644 index 000000000..c28d5b6f3 --- /dev/null +++ b/aicb/workload_generator/workload_generator.py @@ -0,0 +1,87 @@ +""" +Copyright (c) 2021, Alibaba Group; +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from workload_generator.mocked_model.MockedModel import MockedModel +from utils.utils import CommGroup, CommType, RankGenerator +from utils.rank_mapper import get_rank_list_for_comm_group +from log_analyzer.log import Workload, LogItem + + +class WorkloadGenerator: + DEFAULT_ORDER = 'tp-cp-ep-dp-pp' + + def __init__(self, args, model: MockedModel) -> None: + self.name = "workload_generator" + self.args = args + self.model = model + self.workload = Workload() + self.epoch = 0 + + order = getattr(args, 'order', None) or WorkloadGenerator.DEFAULT_ORDER + self.rank_generator = RankGenerator( + tp=args.tensor_model_parallel_size, + ep=getattr(args, 'expert_model_parallel_size', 1), + dp=args.dp_num, + pp=args.pipeline_model_parallel, + cp=getattr(args, 'context_parallel_size', 1), + order=order, + ) + self.workload.rank_generator = self.rank_generator + + def get_ranks(self, comm_group, comm_group_size=None): + return get_rank_list_for_comm_group( + self.rank_generator, comm_group, comm_group_size + ) + + def __call__(self): + args = self.args + self.workload = Workload() + self.workload.rank_generator = self.rank_generator + self.init() + self.workload.append(LogItem(comm_type=CommType.epoch_end)) + for i in range(args.epoch_num): + if args.pipeline_model_parallel > 1 and args.frame != "collective_test": + self.with_pipeline_forward_backward() + self.step() + else: + for _ in range(args.num_microbatches): + self.forward() + self.backward() + self.step() + self.workload.append(LogItem(comm_type=CommType.epoch_end)) + self._fill_ranks() + return self.workload + + def _fill_ranks(self): + for item in self.workload.workload: + if item.ranks is not None: + continue + if item.comm_group is None: + continue + item.ranks = get_rank_list_for_comm_group( + self.rank_generator, item.comm_group, item.comm_group_size + ) + if item.comm_group_size is None and item.ranks: + item.comm_group_size = len(item.ranks) + + def forward(self): + pass + + def backward(self): + pass + + def step(self): + pass + + def with_pipeline_forward_backward(self): + pass